diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 000000000..a8e965838 --- /dev/null +++ b/.bazelignore @@ -0,0 +1,2 @@ +.direnv +build diff --git a/.bazelrc b/.bazelrc index de31dcc06..8f70af802 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,5 +1,5 @@ # Import bazelrc presets -import %workspace%/bazel/bazelrc/bazel6.bazelrc +import %workspace%/bazel/bazelrc/bazel7.bazelrc import %workspace%/bazel/bazelrc/convenience.bazelrc import %workspace%/bazel/bazelrc/correctness.bazelrc import %workspace%/bazel/bazelrc/debug.bazelrc @@ -32,7 +32,7 @@ test --test_tag_filters=-integration # enable all tests (including integration) test:integration --test_tag_filters= --@io_bazel_rules_go//go/config:tags=integration # enable only integration tests -test:integration-only --test_tag_filters=+integration --@io_bazel_rules_go//go/config:tags=integration +test:integration-only --test_tag_filters=+integration --@io_bazel_rules_go//go/config:tags=integration,enterprise # bazel configs to explicitly target a platform common:host --platforms @local_config_platform//:host @@ -48,15 +48,6 @@ common --crosstool_top=@local_config_cc//:toolchain # bazel config to explicitly disable stamping (hide version information at build time) common:nostamp --nostamp --workspace_status_command= -# bazel config to use (buildbuddy) remote cache -common:remote_cache --bes_results_url=https://app.buildbuddy.io/invocation/ -common:remote_cache --bes_backend=grpcs://remote.buildbuddy.io -common:remote_cache --remote_cache=grpcs://remote.buildbuddy.io -common:remote_cache --remote_timeout=3600 -common:remote_cache --experimental_remote_build_event_upload=minimal -common:remote_cache --nolegacy_important_outputs -common:remote_cache_readonly --noremote_upload_local_results # Uploads logs & artifacts without writing to cache - common:build_barn_rbe_ubuntu_22_04 --remote_timeout=3600 common:build_barn_rbe_ubuntu_22_04 --remote_executor=grpc://frontend.buildbarn:8980 # this maps to the kubernetes internal buildbarn/frontend service common:build_barn_rbe_ubuntu_22_04 --extra_execution_platforms=//bazel/rbe:ubuntu-act-22-04-platform diff --git a/.bazelversion b/.bazelversion index 19b860c18..93c8ddab9 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.4.0 +7.6.0 diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..3550a30f2 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.github/actions/artifact_delete/action.yml b/.github/actions/artifact_delete/action.yml new file mode 100644 index 000000000..e2f001621 --- /dev/null +++ b/.github/actions/artifact_delete/action.yml @@ -0,0 +1,17 @@ +name: Delete artifact +description: Delete an artifact by name + +inputs: + name: + description: 'The name of the artifact.' + required: true + workflowID: + description: 'The ID of the workflow.' + required: true + +runs: + using: "composite" + steps: + - name: Delete artifact + shell: bash + run: ./.github/actions/artifact_delete/delete_artifact.sh ${{ inputs.workflowID }} ${{ inputs.name }} diff --git a/.github/actions/artifact_delete/delete_artifact.sh b/.github/actions/artifact_delete/delete_artifact.sh new file mode 100755 index 000000000..942304831 --- /dev/null +++ b/.github/actions/artifact_delete/delete_artifact.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# get_artifact_id retrieves the artifact id of +# an artifact that was generated by a workflow. +# $1 should be the workflow run id. $2 should be the artifact name. +function get_artifact_id { + artifact_id="$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --paginate \ + "/repos/edgelesssys/constellation/actions/runs/$1/artifacts" --jq ".artifacts |= map(select(.name==\"$2\")) | .artifacts[0].id" || exit 1)" + echo "$artifact_id" | tr -d "\n" +} + +# delete_artifact_by_id deletes an artifact by its artifact id. +# $1 should be the id of the artifact. +function delete_artifact_by_id { + gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/edgelesssys/constellation/actions/artifacts/$1" || exit 1 +} + +workflow_id="$1" +artifact_name="$2" + +if [[ -z $workflow_id ]] || [[ -z $artifact_name ]]; then + echo "Usage: delete_artifact.sh " + exit 1 +fi + +echo "[*] retrieving artifact ID" +artifact_id="$(get_artifact_id "$workflow_id" "$artifact_name")" + +echo "[*] deleting artifact with ID $artifact_id" +delete_artifact_by_id "$artifact_id" diff --git a/.github/actions/artifact_download/action.yml b/.github/actions/artifact_download/action.yml new file mode 100644 index 000000000..e3cf3d1f8 --- /dev/null +++ b/.github/actions/artifact_download/action.yml @@ -0,0 +1,40 @@ +name: Download artifact +description: Download and decrypt an artifact. + +inputs: + name: + description: 'The name of the artifact.' + required: true + path: + description: 'Download to a specified path.' + required: false + default: ./ + encryptionSecret: + description: 'The secret to use for decrypting the artifact.' + required: true + +runs: + using: "composite" + steps: + - name: Install 7zip + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: | + _7zz + + - name: Create temporary directory + id: tempdir + shell: bash + run: echo "directory=$(mktemp -d)" >> "$GITHUB_OUTPUT" + + - name: Download the artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: ${{ inputs.name }} + path: ${{ steps.tempdir.outputs.directory }} + + - name: Decrypt and unzip archive + shell: bash + run: | + mkdir -p ${{ inputs.path }} + 7zz x -p'${{ inputs.encryptionSecret }}' -bso0 -bsp0 -t7z -o"${{ inputs.path }}" ${{ steps.tempdir.outputs.directory }}/archive.7z diff --git a/.github/actions/artifact_upload/action.yml b/.github/actions/artifact_upload/action.yml new file mode 100644 index 000000000..2ef3e85a8 --- /dev/null +++ b/.github/actions/artifact_upload/action.yml @@ -0,0 +1,78 @@ +name: Upload artifact +description: Upload an encrypted zip archive as a github artifact. + +inputs: + path: + description: 'The path(s) that should be uploaded. Paths may contain globs. Only the final component of a path is uploaded.' + required: true + name: + description: 'The name of the artifact.' + required: true + retention-days: + description: 'How long the artifact should be retained for.' + default: 60 + encryptionSecret: + description: 'The secret to use for encrypting the files.' + required: true + overwrite: + description: 'Overwrite an artifact with the same name.' + default: false + required: false + +runs: + using: "composite" + steps: + - name: Install 7zip + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: | + _7zz + + - name: Create temporary directory + id: tempdir + shell: bash + run: echo "directory=$(mktemp -d)" >> "$GITHUB_OUTPUT" + + - name: Create archive + shell: bash + run: | + set -euo pipefail + shopt -s extglob + paths="${{ inputs.path }}" + paths=${paths%$'\n'} # Remove trailing newline + # Check if any file matches the given pattern(s). + something_exists=false + for pattern in ${paths} + do + if compgen -G "${pattern}" > /dev/null; then + something_exists=true + fi + done + + # Create an archive if files exist. + # Don't create an archive file if no files are found + # and warn. + if ! ${something_exists} + then + echo "::warning:: No files/directories found with the provided path(s): ${paths}. No artifact will be uploaded." + exit 0 + fi + + for target in ${paths} + do + if compgen -G "${target}" > /dev/null + then + pushd "$(dirname "${target}")" + 7zz a -p'${{ inputs.encryptionSecret }}' -bso0 -bsp0 -t7z -ms=on -mhe=on "${{ steps.tempdir.outputs.directory }}/archive.7z" "$(basename "${target}")" + popd + fi + done + + - name: Upload archive as artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.name }} + path: ${{ steps.tempdir.outputs.directory }}/archive.7z + retention-days: ${{ inputs.retention-days }} + if-no-files-found: ignore + overwrite: ${{ inputs.overwrite }} diff --git a/.github/actions/build_cli/action.yml b/.github/actions/build_cli/action.yml index fd1da938f..b74b67456 100644 --- a/.github/actions/build_cli/action.yml +++ b/.github/actions/build_cli/action.yml @@ -75,11 +75,9 @@ runs: shell: bash run: bazel run //bazel/release:push - # TODO(3u13r): Replace with https://github.com/sigstore/sigstore-installer/tree/initial - # once it has the functionality - name: Install Cosign if: inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' - uses: sigstore/cosign-installer@c85d0e205a72a294fe064f618a87dbac13084086 # v2.8.1 + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 - name: Install Rekor if: inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' @@ -104,7 +102,7 @@ runs: run: | echo "$COSIGN_PUBLIC_KEY" > cosign.pub # Enabling experimental mode also publishes signature to Rekor - COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY "${OUTPUT_PATH}" > "${OUTPUT_PATH}.sig" + COSIGN_EXPERIMENTAL=1 cosign sign-blob --yes --key env://COSIGN_PRIVATE_KEY "${OUTPUT_PATH}" > "${OUTPUT_PATH}.sig" # Verify - As documentation & check # Local Signature (input: artifact, key, signature) cosign verify-blob --key cosign.pub --signature "${OUTPUT_PATH}.sig" "${OUTPUT_PATH}" diff --git a/.github/actions/build_micro_service/action.yml b/.github/actions/build_micro_service/action.yml index aed09c4d9..7fecf16a2 100644 --- a/.github/actions/build_micro_service/action.yml +++ b/.github/actions/build_micro_service/action.yml @@ -42,7 +42,7 @@ runs: - name: Docker metadata id: meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 with: images: | ghcr.io/${{ github.repository }}/${{ inputs.name }} @@ -62,7 +62,7 @@ runs: - name: Build and push container image id: build-micro-service - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: . file: ${{ inputs.dockerfile }} diff --git a/.github/actions/cdbg_deploy/action.yml b/.github/actions/cdbg_deploy/action.yml index d2c7f4140..a51c54b6a 100644 --- a/.github/actions/cdbg_deploy/action.yml +++ b/.github/actions/cdbg_deploy/action.yml @@ -14,6 +14,9 @@ inputs: cloudProvider: description: "The cloud provider to use." required: true + attestationVariant: + description: "Attestation variant of the cluster." + required: false kubernetesVersion: description: "Kubernetes version to create the cluster from." required: true @@ -21,7 +24,7 @@ inputs: description: "The refStream of the image the test runs on." required: true clusterCreation: - description: "How the infrastructure for the e2e test was created. One of [cli, self-managed, terraform]." + description: "How the infrastructure for the e2e test was created. One of [cli, terraform]." default: "cli" runs: @@ -37,8 +40,15 @@ runs: if: inputs.cloudProvider == 'azure' shell: bash run: | - UAMI=$(yq eval ".provider.azure.userAssignedIdentity | upcase" constellation-conf.yaml) - PRINCIPAL_ID=$(az identity list | yq ".[] | select(.id | test(\"(?i)$UAMI\"; \"g\")) | .principalId") + UAMI=$(yq eval ".provider.azure.userAssignedIdentity" constellation-conf.yaml) + PRINCIPAL_ID=$(az identity show --ids "$UAMI" | yq ".principalId") + if [ -z "$PRINCIPAL_ID" ]; then + echo "::error::PRINCIPAL_ID for \"$UAMI\" not found" + echo "::group::Available identities" + az identity list | yq ".[].id" + echo "::endgroup::" + exit 1 + fi az role assignment create --role "Key Vault Secrets User" \ --assignee "$PRINCIPAL_ID" \ --scope /subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/e2e-test-creds/providers/Microsoft.KeyVault/vaults/opensearch-creds @@ -51,7 +61,7 @@ runs: - name: Login to AWS (IAM service principal) if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM aws-region: eu-central-1 @@ -70,7 +80,7 @@ runs: - name: Login to AWS (Cluster service principal) if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster aws-region: eu-central-1 @@ -81,6 +91,11 @@ runs: shell: bash run: | echo "::group::cdbg deploy" + on_error() { + echo "::error::cdbg deploy failed" + } + trap on_error ERR + chmod +x $GITHUB_WORKSPACE/build/cdbg cdbg deploy \ --bootstrapper "${{ github.workspace }}/build/bootstrapper" \ @@ -98,6 +113,7 @@ runs: --info logcollect.github.ref-stream="${{ inputs.refStream }}" \ --info logcollect.github.kubernetes-version="${{ inputs.kubernetesVersion }}" \ --info logcollect.github.cluster-creation="${{ inputs.clusterCreation }}" \ + --info logcollect.github.attestation-variant="${{ inputs.attestationVariant }}" \ --info logcollect.deployment-type="debugd" \ --verbosity=-1 \ --force diff --git a/.github/actions/check_measurements_reproducibility/action.yml b/.github/actions/check_measurements_reproducibility/action.yml new file mode 100644 index 000000000..184e1221f --- /dev/null +++ b/.github/actions/check_measurements_reproducibility/action.yml @@ -0,0 +1,64 @@ +name: Check measurements reproducibility +description: Check if the measurements of a given release are reproducible. + +inputs: + version: + type: string + description: The version of the measurements that are downloaded from the CDN. + required: true + ref: + type: string + description: The git ref to check out. You probably want this to be the tag of the release you are testing. + required: true + +runs: + using: "composite" + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref }} + path: ./release + + - name: Set up bazel + uses: ./.github/actions/setup_bazel_nix + with: + useCache: "false" + nixTools: | + systemdUkify + jq + jd-diff-patch + moreutils + + - name: Allow unrestricted user namespaces + shell: bash + run: | + sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_unconfined=0 + sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Build images + id: build-images + shell: bash + run: | + set -euo pipefail + + # Build required binaries + pushd release + bazel build //image/system:stable + echo "buildPath=$PWD/bazel-bin/image" | tee -a "$GITHUB_OUTPUT" + popd + + - name: Download measurements + shell: bash + run: | + curl -fsLO https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/${{ inputs.version }}/image/measurements.json + + - name: Cleanup release measurements and generate our own + shell: bash + run: | + ${{ github.action_path }}/create_measurements.sh "${{ steps.build-images.outputs.buildPath }}" + + - name: Compare measurements + shell: bash + run: | + ${{ github.action_path }}/compare_measurements.sh "${{ steps.build-images.outputs.buildPath }}" diff --git a/.github/actions/check_measurements_reproducibility/compare_measurements.sh b/.github/actions/check_measurements_reproducibility/compare_measurements.sh new file mode 100755 index 000000000..5077a05f2 --- /dev/null +++ b/.github/actions/check_measurements_reproducibility/compare_measurements.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# no -e since we need to collect errors later +# no -u since it interferes with checking associative arrays +set -o pipefail +shopt -s extglob + +declare -A errors + +for directory in "$1"/system/!(mkosi_wrapper.sh); do + dirname="$(basename "$directory")" + attestationVariant="$(echo "$dirname" | cut -d_ -f2)" + + echo "Their measurements for $attestationVariant:" + ts " " < "$attestationVariant"_their-measurements.json + echo "Own measurements for $attestationVariant:" + ts " " < "$attestationVariant"_own-measurements.json + + diff="$(jd ./"$attestationVariant"_their-measurements.json ./"$attestationVariant"_own-measurements.json)" + if [[ -n $diff ]]; then + errors["$attestationVariant"]="$diff" + fi +done + +for attestationVariant in "${!errors[@]}"; do + echo "Failed to reproduce measurements for $attestationVariant:" + echo "${errors["$attestationVariant"]}" | ts " " +done + +if [[ ${#errors[@]} -ne 0 ]]; then + exit 1 +fi diff --git a/.github/actions/check_measurements_reproducibility/create_measurements.sh b/.github/actions/check_measurements_reproducibility/create_measurements.sh new file mode 100755 index 000000000..4cabd5df0 --- /dev/null +++ b/.github/actions/check_measurements_reproducibility/create_measurements.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s extglob + +for directory in "$1"/system/!(mkosi_wrapper.sh); do + dirname="$(basename "$directory")" + csp="$(echo "$dirname" | cut -d_ -f1)" + attestationVariant="$(echo "$dirname" | cut -d_ -f2)" + + # This jq filter selects the measurements for the correct CSP and attestation variant + # and then removes all `warnOnly: true` measurements. + jq --arg attestation_variant "$attestationVariant" --arg csp "$csp" \ + ' + .list.[] + | select( + .attestationVariant == $attestation_variant + and (.csp | ascii_downcase) == $csp + ) + | .measurements + | to_entries + | map(select(.value.warnOnly | not)) + | from_entries + | del(.[] .warnOnly) + ' \ + measurements.json > "$attestationVariant"_their-measurements.json + + bazel run --run_under "sudo --preserve-env" //image/measured-boot/cmd -- "$directory/constellation" /dev/stdout | jq '.measurements' > ./"$attestationVariant"_own-measurements.json +done diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml index 6e72455fe..caec827d6 100644 --- a/.github/actions/constellation_create/action.yml +++ b/.github/actions/constellation_create/action.yml @@ -11,6 +11,9 @@ inputs: cloudProvider: description: "Either 'gcp', 'aws' or 'azure'." required: true + attestationVariant: + description: "Attestation variant to use." + required: true machineType: description: "Machine type of VM to spawn." required: false @@ -51,7 +54,7 @@ inputs: description: "Whether to use an internal load balancer for the control plane" required: false clusterCreation: - description: "How to create infrastructure for the e2e test. One of [cli, self-managed, terraform]." + description: "How to create infrastructure for the e2e test. One of [cli, terraform]." default: "cli" marketplaceImageVersion: description: "Marketplace OS image version. Used instead of osImage." @@ -59,6 +62,9 @@ inputs: force: description: "Set the force-flag on apply to ignore version mismatches." required: false + encryptionSecret: + description: "The secret to use for encrypting the artifact." + required: true outputs: kubeconfig: @@ -80,7 +86,7 @@ runs: if: inputs.azureSNPEnforcementPolicy != '' shell: bash run: | - if [[ ${{ inputs.cloudProvider }} != 'azure' ]]; then + if [[ ${{ inputs.attestationVariant }} != 'azure-sev-snp' ]]; then echo "SNP enforcement policy is only supported for Azure" exit 1 fi @@ -103,6 +109,13 @@ runs: yq eval -i "(.image) = \"${imageInput}\"" constellation-conf.yaml echo "image=${imageInput}" | tee -a "$GITHUB_OUTPUT" + - name: Set marketplace image flag (AWS) + if: inputs.marketplaceImageVersion != '' && inputs.cloudProvider == 'aws' + shell: bash + run: | + yq eval -i "(.provider.aws.useMarketplaceImage) = true" constellation-conf.yaml + yq eval -i "(.image) = \"${{ inputs.marketplaceImageVersion }}\"" constellation-conf.yaml + - name: Set marketplace image flag (Azure) if: inputs.marketplaceImageVersion != '' && inputs.cloudProvider == 'azure' shell: bash @@ -110,6 +123,13 @@ runs: yq eval -i "(.provider.azure.useMarketplaceImage) = true" constellation-conf.yaml yq eval -i "(.image) = \"${{ inputs.marketplaceImageVersion }}\"" constellation-conf.yaml + - name: Set marketplace image flag (GCP) + if: inputs.marketplaceImageVersion != '' && inputs.cloudProvider == 'gcp' + shell: bash + run: | + yq eval -i "(.provider.gcp.useMarketplaceImage) = true" constellation-conf.yaml + yq eval -i "(.image) = \"${{ inputs.marketplaceImageVersion }}\"" constellation-conf.yaml + - name: Update measurements for non-stable images if: inputs.fetchMeasurements shell: bash @@ -148,27 +168,16 @@ runs: sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' || true - name: Constellation create (CLI) - if : inputs.clusterCreation != 'self-managed' shell: bash run: | - # TODO(v2.14): Remove workaround for CLIs not supporting apply command - cmd='apply --skip-phases=init,attestationconfig,certsans,helm,image,k8s' - if constellation --help | grep -q create; then - cmd=create - fi - constellation $cmd -y --debug --tf-log=DEBUG - - - name: Constellation create (self-managed) - if : inputs.clusterCreation == 'self-managed' - uses: ./.github/actions/self_managed_create - with: - cloudProvider: ${{ inputs.cloudProvider }} + constellation apply --skip-phases=init,attestationconfig,certsans,helm,image,k8s -y --debug --tf-log=DEBUG - name: Cdbg deploy if: inputs.isDebugImage == 'true' uses: ./.github/actions/cdbg_deploy with: cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} test: ${{ inputs.test }} azureClusterCreateCredentials: ${{ inputs.azureClusterCreateCredentials }} azureIAMCreateCredentials: ${{ inputs.azureIAMCreateCredentials }} @@ -183,6 +192,13 @@ runs: run: | echo "flag=--force" | tee -a $GITHUB_OUTPUT + - name: Set conformance flag + id: set-conformance-flag + if: inputs.test == 'sonobuoy conformance' + shell: bash + run: | + echo "flag=--conformance" | tee -a $GITHUB_OUTPUT + - name: Constellation apply (Terraform) id: constellation-apply-terraform if: inputs.clusterCreation == 'terraform' @@ -195,7 +211,7 @@ runs: if: inputs.clusterCreation != 'terraform' shell: bash run: | - constellation apply --skip-phases=infrastructure --debug ${{ steps.set-force-flag.outputs.flag }} + constellation apply --skip-phases=infrastructure --debug ${{ steps.set-force-flag.outputs.flag }} ${{ steps.set-conformance-flag.outputs.flag }} - name: Get kubeconfig id: get-kubeconfig @@ -208,29 +224,9 @@ runs: env: KUBECONFIG: "${{ steps.get-kubeconfig.outputs.KUBECONFIG }}" JOINTIMEOUT: "1200" # 20 minutes timeout for all nodes to join - run: | - echo "::group::Wait for nodes" - NODES_COUNT=$((${{ inputs.controlNodesCount }} + ${{ inputs.workerNodesCount }})) - JOINWAIT=0 - until [[ "$(kubectl get nodes -o json | jq '.items | length')" == "${NODES_COUNT}" ]] || [[ $JOINWAIT -gt $JOINTIMEOUT ]]; - do - echo "$(kubectl get nodes -o json | jq '.items | length')/"${NODES_COUNT}" nodes have joined.. waiting.." - JOINWAIT=$((JOINWAIT+30)) - sleep 30 - done - if [[ $JOINWAIT -gt $JOINTIMEOUT ]]; then - echo "Timed out waiting for nodes to join" - exit 1 - fi - echo "$(kubectl get nodes -o json | jq '.items | length')/"${NODES_COUNT}" nodes have joined" - if ! kubectl wait --for=condition=ready --all nodes --timeout=20m; then - kubectl get pods -n kube-system - kubectl get events -n kube-system - echo "::error::kubectl wait timed out before all nodes became ready" - echo "::endgroup::" - exit 1 - fi - echo "::endgroup::" + CONTROL_NODES_COUNT: "${{ inputs.controlNodesCount }}" + WORKER_NODES_COUNT: "${{ inputs.workerNodesCount }}" + run: ./.github/actions/constellation_create/wait-for-nodes.sh - name: Download boot logs if: always() @@ -259,9 +255,32 @@ runs: - name: Upload boot logs if: always() && !env.ACT continue-on-error: true - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: ./.github/actions/artifact_upload with: - name: serial-logs-${{ inputs.artifactNameSuffix }} + name: debug-logs-${{ inputs.artifactNameSuffix }} path: | *.log - !terraform.log + encryptionSecret: ${{ inputs.encryptionSecret }} + + - name: Prepare terraform state folders + if: always() + shell: bash + run: | + mkdir to-zip + cp -r constellation-terraform to-zip + # constellation-iam-terraform is optional + if [ -d constellation-iam-terraform ]; then + cp -r constellation-iam-terraform to-zip + fi + rm -f to-zip/constellation-terraform/plan.zip + rm -rf to-zip/*/.terraform + + - name: Upload terraform state + if: always() + uses: ./.github/actions/artifact_upload + with: + name: terraform-state-${{ inputs.artifactNameSuffix }} + path: > + to-zip/constellation-terraform + to-zip/constellation-iam-terraform + encryptionSecret: ${{ inputs.encryptionSecret }} diff --git a/.github/actions/constellation_create/aws-logs.sh b/.github/actions/constellation_create/aws-logs.sh index 483ab33ca..768e4446b 100755 --- a/.github/actions/constellation_create/aws-logs.sh +++ b/.github/actions/constellation_create/aws-logs.sh @@ -25,20 +25,20 @@ workerInstances=$( yq eval '.Reservations[].Instances[].InstanceId' - ) -echo "Fetching logs from control planes" +for flag in "" "--latest"; do + echo "Fetching ${flag} logs from control planes" + for instance in ${controlInstances}; do + printf "Fetching for %s\n" "${instance}" + aws ec2 get-console-output "${flag}" --region "${1}" --instance-id "${instance}" | + jq -r .'Output' | + tail -n +2 > "control-plane-${instance}${flag}.log" + done -for instance in ${controlInstances}; do - printf "Fetching for %s\n" "${instance}" - aws ec2 get-console-output --region "${1}" --instance-id "${instance}" | - jq -r .'Output' | - tail -n +2 > control-plane-"${instance}".log -done - -echo "Fetching logs from worker nodes" - -for instance in ${workerInstances}; do - printf "Fetching for %s\n" "${instance}" - aws ec2 get-console-output --region "${1}" --instance-id "${instance}" | - jq -r .'Output' | - tail -n +2 > worker-"${instance}".log + echo "Fetching ${flag} logs from worker nodes" + for instance in ${workerInstances}; do + printf "Fetching for %s\n" "${instance}" + aws ec2 get-console-output "${flag}" --region "${1}" --instance-id "${instance}" | + jq -r .'Output' | + tail -n +2 > "worker-${instance}${flag}.log" + done done diff --git a/.github/actions/constellation_create/wait-for-nodes.sh b/.github/actions/constellation_create/wait-for-nodes.sh new file mode 100755 index 000000000..9fb9b36e4 --- /dev/null +++ b/.github/actions/constellation_create/wait-for-nodes.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# We don't want to abort the script if there's a transient error in kubectl. +set +e +set -uo pipefail + +NODES_COUNT=$((CONTROL_NODES_COUNT + WORKER_NODES_COUNT)) +JOINWAIT=0 + +# Reports how many nodes are registered and fulfill condition=ready. +num_nodes_ready() { + kubectl get nodes -o json | + jq '.items | map(select(.status.conditions[] | .type == "Ready" and .status == "True")) | length' +} + +# Reports how many API server pods are ready. +num_apiservers_ready() { + kubectl get pods -n kube-system -l component=kube-apiserver -o json | + jq '.items | map(select(.status.conditions[] | .type == "Ready" and .status == "True")) | length' +} + +# Prints node joining progress. +report_join_progress() { + echo -n "nodes_joined=$(kubectl get nodes -o json | jq '.items | length')/${NODES_COUNT} " + echo -n "nodes_ready=$(num_nodes_ready)/${NODES_COUNT} " + echo "api_servers_ready=$(num_apiservers_ready)/${CONTROL_NODES_COUNT} ..." +} + +# Indicates by exit code whether the cluster is ready, i.e. all nodes and API servers are ready. +cluster_ready() { + [[ "$(num_nodes_ready)" == "${NODES_COUNT}" && "$(num_apiservers_ready)" == "${CONTROL_NODES_COUNT}" ]] +} + +echo "::group::Wait for nodes" +until cluster_ready || [[ ${JOINWAIT} -gt ${JOINTIMEOUT} ]]; do + report_join_progress + JOINWAIT=$((JOINWAIT + 30)) + sleep 30 +done +report_join_progress +if [[ ${JOINWAIT} -gt ${JOINTIMEOUT} ]]; then + set -x + kubectl get nodes -o wide + kubectl get pods -n kube-system -o wide + kubectl get events -n kube-system + set +x + echo "::error::timeout reached before all nodes became ready" + echo "::endgroup::" + exit 1 +fi +echo "::endgroup::" diff --git a/.github/actions/constellation_destroy/action.yml b/.github/actions/constellation_destroy/action.yml index 0d56fd89d..c09148efa 100644 --- a/.github/actions/constellation_destroy/action.yml +++ b/.github/actions/constellation_destroy/action.yml @@ -6,7 +6,7 @@ inputs: description: "The kubeconfig for the cluster." required: true clusterCreation: - description: "How the infrastructure for the e2e test was created. One of [cli, self-managed, terraform]." + description: "How the infrastructure for the e2e test was created. One of [cli, terraform]." default: "cli" gcpClusterDeleteServiceAccount: description: "Service account with permissions to delete a Constellation cluster on GCP." @@ -24,6 +24,7 @@ runs: - name: Delete persistent volumes if: inputs.kubeconfig != '' shell: bash + continue-on-error: true env: KUBECONFIG: ${{ inputs.kubeconfig }} PV_DELETION_TIMEOUT: "120" # 2 minutes timeout for pv deletion @@ -34,6 +35,14 @@ runs: # Scrap namespaces that contain PVCs for namespace in `kubectl get namespace --no-headers=true -o custom-columns=":metadata.name"`; do if [[ `kubectl get pvc -n $namespace --no-headers=true -o custom-columns=":metadata.name" | wc -l` -gt 0 ]]; then + if [[ "${namespace}" == "default" ]]; then + kubectl delete all --all --namespace "default" --wait + continue + fi + if [[ "${namespace}" == "kube-system" ]]; then + kubectl delete pvc --all --namespace "kube-system" --wait + continue + fi kubectl delete namespace $namespace --wait fi done @@ -58,7 +67,7 @@ runs: - name: Login to AWS (Cluster role) if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster aws-region: eu-central-1 @@ -72,18 +81,6 @@ runs: azure_credentials: ${{ inputs.azureClusterDeleteCredentials }} - name: Constellation terminate - if: inputs.clusterCreation != 'self-managed' shell: bash run: | constellation terminate --yes --tf-log=DEBUG - - - name: Constellation terminate (self-managed) - if: inputs.clusterCreation == 'self-managed' - shell: bash - working-directory: ${{ github.workspace }}/e2e-infra - run: | - terraform init - terraform destroy -auto-approve - - rm -f ${{ github.workspace }}/constellation-state.yaml - rm -f ${{ github.workspace }}/constellation-admin.conf diff --git a/.github/actions/constellation_iam_create/action.yml b/.github/actions/constellation_iam_create/action.yml index f5f307e88..46c5ef939 100644 --- a/.github/actions/constellation_iam_create/action.yml +++ b/.github/actions/constellation_iam_create/action.yml @@ -5,12 +5,19 @@ inputs: cloudProvider: description: "Either 'aws', 'azure' or 'gcp'." required: true + attestationVariant: + description: "The attestation variant to use." + required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." required: false namePrefix: description: "Name prefix to use for resources." required: true + additionalTags: + description: "Additional resource tags that will be written into the constellation configuration." + default: "" + required: false # # AWS specific inputs # @@ -20,6 +27,9 @@ inputs: # # Azure specific inputs # + azureSubscriptionID: + description: "Azure subscription ID to deploy Constellation in." + required: true azureRegion: description: "Azure region to deploy Constellation in." required: false @@ -32,6 +42,15 @@ inputs: gcpZone: description: "The GCP zone to deploy Constellation in." required: false + # + # STACKIT specific inputs + # + stackitZone: + description: "The STACKIT zone to deploy Constellation in." + required: false + stackitProjectID: + description: "The STACKIT project ID to deploy Constellation in." + required: false runs: using: "composite" @@ -45,8 +64,14 @@ runs: kubernetesFlag="--kubernetes=${{ inputs.kubernetesVersion }}" fi + # TODO(v2.17): Remove this fallback and always use --tags flag + tagsFlag="" + if constellation config generate --help | grep -q -- --tags; then + tagsFlag="--tags=\"${{ inputs.additionalTags }}\"" + fi + echo "flag=--update-config" | tee -a "$GITHUB_OUTPUT" - constellation config generate ${{ inputs.cloudProvider }} ${kubernetesFlag} + constellation config generate ${{ inputs.cloudProvider }} ${kubernetesFlag} --attestation ${{ inputs.attestationVariant }} ${tagsFlag} - name: Constellation iam create aws shell: bash @@ -63,14 +88,21 @@ runs: shell: bash if: inputs.cloudProvider == 'azure' run: | + extraFlags="" + + if [[ $(constellation iam create azure --help | grep -c -- --subscriptionID) -ne 0 ]]; then + extraFlags="--subscriptionID=${{ inputs.azureSubscriptionID }}" + fi + constellation iam create azure \ --region="${{ inputs.azureRegion }}" \ --resourceGroup="${{ inputs.namePrefix }}-rg" \ --servicePrincipal="${{ inputs.namePrefix }}-sp" \ --update-config \ --tf-log=DEBUG \ - --yes + --yes ${extraFlags} + # TODO: Replace deprecated --serviceAccountID with --prefix - name: Constellation iam create gcp shell: bash if: inputs.cloudProvider == 'gcp' @@ -82,3 +114,13 @@ runs: --update-config \ --tf-log=DEBUG \ --yes + + - name: Set STACKIT-specific configuration + shell: bash + if: inputs.cloudProvider == 'stackit' + env: + STACKIT_PROJECT_ID: ${{ inputs.stackitProjectID }} + run: | + yq eval -i "(.provider.openstack.stackitProjectID) = \"${STACKIT_PROJECT_ID}\"" constellation-conf.yaml + yq eval -i "(.provider.openstack.availabilityZone) = \"${{ inputs.stackitZone }}\"" constellation-conf.yaml + yq eval -i "(.nodeGroups.[].zone) = \"${{ inputs.stackitZone }}\"" constellation-conf.yaml diff --git a/.github/actions/constellation_iam_destroy/action.yml b/.github/actions/constellation_iam_destroy/action.yml index fdc28d82c..98109d740 100644 --- a/.github/actions/constellation_iam_destroy/action.yml +++ b/.github/actions/constellation_iam_destroy/action.yml @@ -23,7 +23,7 @@ runs: - name: Login to AWS (IAM role) if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM aws-region: eu-central-1 diff --git a/.github/actions/container_registry_login/action.yml b/.github/actions/container_registry_login/action.yml index fd3b7f230..929af6361 100644 --- a/.github/actions/container_registry_login/action.yml +++ b/.github/actions/container_registry_login/action.yml @@ -17,7 +17,7 @@ runs: steps: - name: Use docker for logging in if: runner.os != 'macOS' - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ${{ inputs.registry }} username: ${{ inputs.username }} diff --git a/.github/actions/container_sbom/action.yml b/.github/actions/container_sbom/action.yml index bea12d954..0d259003d 100644 --- a/.github/actions/container_sbom/action.yml +++ b/.github/actions/container_sbom/action.yml @@ -19,7 +19,7 @@ runs: steps: - name: Install Cosign if: inputs.cosignPublicKey != '' && inputs.cosignPrivateKey != '' && inputs.cosignPassword != '' - uses: sigstore/cosign-installer@c85d0e205a72a294fe064f618a87dbac13084086 # v2.8.1 + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 - name: Download Syft & Grype uses: ./.github/actions/install_syft_grype @@ -36,7 +36,7 @@ runs: syft packages ${{ inputs.containerReference }} -o cyclonedx-json > container-image-predicate.json cosign attest ${{ inputs.containerReference }} --key env://COSIGN_PRIVATE_KEY --predicate container-image-predicate.json --type "https://cyclonedx.org/bom" > container-image.att.json cosign attach attestation ${{ inputs.containerReference }} --attestation container-image.att.json - # TODO(3u13r): type should be auto-discovered after issue is resolved: + # TODO: type should be auto-discovered after issue is resolved: # https://github.com/sigstore/cosign/issues/2264 cosign verify-attestation ${{ inputs.containerReference }} --type "https://cyclonedx.org/bom" --key env://COSIGN_PUBLIC_KEY grype ${{ inputs.containerReference }} --fail-on high --only-fixed --add-cpes-if-none diff --git a/.github/actions/deploy_logcollection/action.yml b/.github/actions/deploy_logcollection/action.yml index 09b63886a..78abb5146 100644 --- a/.github/actions/deploy_logcollection/action.yml +++ b/.github/actions/deploy_logcollection/action.yml @@ -20,6 +20,9 @@ inputs: provider: description: "The CSP of the cluster." required: true + attestationVariant: + description: "Attestation variant of the cluster." + required: false isDebugImage: description: "Whether the cluster is a debug cluster / uses a debug image." required: true @@ -30,7 +33,7 @@ inputs: description: "Kubernetes version of the cluster" required: false clusterCreation: - description: "How the infrastructure for the e2e test was created. One of [cli, self-managed, terraform]." + description: "How the infrastructure for the e2e test was created. One of [cli, terraform]." default: "cli" runs: @@ -58,14 +61,15 @@ runs: --fields github.ref-stream="${{ inputs.refStream }}" \ --fields github.kubernetes-version="${{ inputs.kubernetesVersion }}" \ --fields github.cluster-creation="${{ inputs.clusterCreation }}" \ + --fields github.attestation-variant="${{ inputs.attestationVariant }}" \ --fields deployment-type="k8s" # Make sure that helm is installed # This is not always the case, e.g. on MacOS runners - name: Install Helm - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 + uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 with: - version: latest + version: v3.9.0 - name: Deploy Logstash id: deploy-logstash @@ -90,17 +94,3 @@ runs: helm repo update helm install filebeat elastic/filebeat \ --wait --timeout=1200s --values values.yml - - - name: Deploy Metricbeat - id: deploy-metricbeat - shell: bash - working-directory: ./metricbeat - env: - KUBECONFIG: ${{ inputs.kubeconfig }} - run: | - helm repo add elastic https://helm.elastic.co - helm repo update - helm install metricbeat-k8s elastic/metricbeat \ - --wait --timeout=1200s --values values-control-plane.yml - helm install metricbeat-system elastic/metricbeat \ - --wait --timeout=1200s --values values-all-nodes.yml diff --git a/.github/actions/download_release_binaries/action.yml b/.github/actions/download_release_binaries/action.yml index af8cfa398..6b5604c24 100644 --- a/.github/actions/download_release_binaries/action.yml +++ b/.github/actions/download_release_binaries/action.yml @@ -5,51 +5,51 @@ runs: using: "composite" steps: - name: Download CLI binaries darwin-amd64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation-darwin-amd64 - name: Download CLI binaries darwin-arm64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation-darwin-arm64 - name: Download CLI binaries linux-amd64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation-linux-amd64 - name: Download CLI binaries linux-arm64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation-linux-arm64 - name: Download CLI binaries windows-amd64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation-windows-amd64 - name: Download Terraform module - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: terraform-module - name: Download Terraform provider binary darwin-amd64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: terraform-provider-constellation-darwin-amd64 - name: Download Terraform provider binary darwin-arm64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: terraform-provider-constellation-darwin-arm64 - name: Download Terraform provider binary linux-amd64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: terraform-provider-constellation-linux-amd64 - name: Download Terraform provider binary linux-arm64 - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: terraform-provider-constellation-linux-arm64 diff --git a/.github/actions/e2e_attestationconfigapi/action.yml b/.github/actions/e2e_attestationconfigapi/action.yml index add289476..9ffad3040 100644 --- a/.github/actions/e2e_attestationconfigapi/action.yml +++ b/.github/actions/e2e_attestationconfigapi/action.yml @@ -2,12 +2,9 @@ name: E2E Attestationconfig API Test description: "Test the attestationconfig CLI is functional." inputs: - csp: - description: "Cloud provider to run tests against" - default: "azure" - buildBuddyApiKey: - description: "BuildBuddy API key for caching Bazel artifacts" - required: true + attestationVariant: + description: "attestation variant to run tests against" + default: "azure-sev-snp" cosignPrivateKey: description: "Cosign private key" required: true @@ -20,12 +17,9 @@ runs: steps: - name: Setup bazel uses: ./.github/actions/setup_bazel_nix - with: - useCache: "true" - buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }} - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubTestResourceAPI aws-region: eu-west-1 @@ -36,4 +30,4 @@ runs: COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }} COSIGN_PASSWORD: ${{ inputs.cosignPassword }} run: | - bazel run //internal/api/attestationconfigapi/cli:cli_e2e_test -- ${{ inputs.csp }} + bazel run //internal/api/attestationconfigapi/cli:cli_e2e_test -- ${{ inputs.attestationVariant }} diff --git a/.github/actions/e2e_autoscaling/action.yml b/.github/actions/e2e_autoscaling/action.yml index bf787100b..96c9907ed 100644 --- a/.github/actions/e2e_autoscaling/action.yml +++ b/.github/actions/e2e_autoscaling/action.yml @@ -82,7 +82,30 @@ runs: KUBECONFIG: ${{ inputs.kubeconfig }} run: | worker_count=${{ steps.worker_count.outputs.worker_count }} - kubectl create -n default deployment nginx --image=nginx --replicas $(( 110 * (worker_count + 1) + 55 )) + + cat < + benchmarks/constellation-${{ inputs.attestationVariant }}.json + name: "benchmarks-${{ inputs.artifactNameSuffix }}" + encryptionSecret: ${{ inputs.encryptionSecret }} - name: Assume AWS role to retrieve and update benchmarks in S3 - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionUpdateBenchmarks aws-region: us-east-2 @@ -133,46 +179,27 @@ runs: - name: Get previous benchmark records from S3 shell: bash - env: - CSP: ${{ inputs.cloudProvider }} run: | - mkdir -p benchmarks - aws s3 cp --recursive ${S3_PATH} benchmarks --no-progress - if [[ -f benchmarks/constellation-${CSP}.json ]]; then - mv benchmarks/constellation-${CSP}.json benchmarks/constellation-${CSP}-previous.json + if aws s3 cp "${S3_PATH}/constellation-${{ inputs.attestationVariant }}.json" ./ --no-progress + then + mv "constellation-${{ inputs.attestationVariant }}.json" "benchmarks/constellation-${{ inputs.attestationVariant }}-previous.json" else echo "::warning::Couldn't retrieve previous benchmark records from s3" fi - - name: Parse results, create diagrams and post the progression summary + - name: Compare results shell: bash env: - # Original result directory - BENCH_RESULTS: out/ - # Working directory containing the previous results as JSON and to contain the graphs - BDIR: benchmarks # Paths to benchmark results as JSON of the previous run and the current run - PREV_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}-previous.json - CURR_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}.json - CSP: ${{ inputs.cloudProvider }} + PREV_BENCH: benchmarks/constellation-${{ inputs.attestationVariant }}-previous.json + CURR_BENCH: benchmarks/constellation-${{ inputs.attestationVariant }}.json run: | - python .github/actions/e2e_benchmark/evaluate/parse.py - export BENCHMARK_SUCCESS=true if [[ -f "$PREV_BENCH" ]]; then - # Sets $BENCHMARK_SUCCESS=false if delta is bigger than defined in compare.py + # Fails if the results are outside the threshold range python .github/actions/e2e_benchmark/evaluate/compare.py >> $GITHUB_STEP_SUMMARY fi - echo BENCHMARK_SUCCESS=$BENCHMARK_SUCCESS >> $GITHUB_ENV - - name: Upload benchmark results to action run - if: (!env.ACT) - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - path: | - benchmarks/constellation-${{ inputs.cloudProvider }}.json - name: "benchmarks" - - - name: Upload benchmark results to opensearch + - name: Upload benchmark results to OpenSearch if: (!env.ACT) shell: bash env: @@ -182,24 +209,12 @@ runs: run: | curl -XPOST \ -u "${OPENSEARCH_USER}:${OPENSEARCH_PWD}" \ - "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.cloudProvider }}-$(date '+%Y')"/_doc \ - --data-binary @benchmarks/constellation-${{ inputs.cloudProvider }}.json \ + "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.attestationVariant }}-$(date '+%Y')"/_doc \ + --data-binary @benchmarks/constellation-${{ inputs.attestationVariant }}.json \ -H 'Content-Type: application/json' - name: Update benchmark records in S3 if: github.ref_name == 'main' shell: bash - env: - CSP: ${{ inputs.cloudProvider }} run: | - aws s3 cp benchmarks/constellation-${CSP}.json ${S3_PATH}/constellation-${CSP}.json - - - name: Check performance comparison result - shell: bash - run: | - if [[ $BENCHMARK_SUCCESS == true ]] ; then - echo "Benchmark successful, all metrics in the expected range." - else - echo "::error::Benchmark failed, some metrics are outside of the expected range." - exit 1 - fi + aws s3 cp benchmarks/constellation-${{ inputs.attestationVariant }}.json ${S3_PATH}/constellation-${{ inputs.attestationVariant }}.json diff --git a/.github/actions/e2e_benchmark/evaluate/compare.py b/.github/actions/e2e_benchmark/evaluate/compare.py index 3d65b0daa..87faac09e 100644 --- a/.github/actions/e2e_benchmark/evaluate/compare.py +++ b/.github/actions/e2e_benchmark/evaluate/compare.py @@ -40,12 +40,15 @@ API_UNIT_STR = "ms" # List of allowed deviation ALLOWED_RATIO_DELTA = { - 'iops': 0.7, - 'bw_kbytes': 0.7, - 'tcp_bw_mbit': 0.7, - 'udp_bw_mbit': 0.7, + 'iops': 0.8, + 'bw_kbytes': 0.8, + 'tcp_bw_mbit': 0.8, + 'udp_bw_mbit': 0.8, } +# Track failed comparison status +failed = False + def is_bigger_better(bench_suite: str) -> bool: return bench_suite in BIGGER_BETTER @@ -91,18 +94,18 @@ class BenchmarkComparer: raise ValueError('Failed reading benchmark file: {e}'.format(e=e)) try: - name = bench_curr['provider'] + name = bench_curr['attestationVariant'] except KeyError: raise ValueError( - 'Current benchmark record file does not contain provider.') + 'Current benchmark record file does not contain attestationVariant.') try: - prev_name = bench_prev['provider'] + prev_name = bench_prev['attestationVariant'] except KeyError: raise ValueError( - 'Previous benchmark record file does not contain provider.') + 'Previous benchmark record file does not contain attestationVariant.') if name != prev_name: raise ValueError( - 'Cloud providers of previous and current benchmark data do not match.') + 'Cloud attestationVariants of previous and current benchmark data do not match.') if 'fio' not in bench_prev.keys() or 'fio' not in bench_curr.keys(): raise ValueError('Benchmarks do not both contain fio records.') @@ -171,7 +174,8 @@ class BenchmarkComparer: def set_failed() -> None: - os.environ['COMPARISON_SUCCESS'] = str(False) + global failed + failed = True def main(): @@ -179,6 +183,8 @@ def main(): c = BenchmarkComparer(path_prev, path_curr) output = c.compare() print(output) + if failed: + exit(1) if __name__ == '__main__': diff --git a/.github/actions/e2e_benchmark/evaluate/parse.py b/.github/actions/e2e_benchmark/evaluate/parse.py index fedce5c70..8d9353343 100644 --- a/.github/actions/e2e_benchmark/evaluate/parse.py +++ b/.github/actions/e2e_benchmark/evaluate/parse.py @@ -7,7 +7,7 @@ from datetime import datetime from evaluators import fio, knb -def configure() -> Tuple[str, str, str, str | None, str, str, str, str]: +def configure() -> Tuple[str, str, str, str, str | None, str, str, str, str]: """Read the benchmark data paths. Expects ENV vars (required): @@ -25,27 +25,29 @@ def configure() -> Tuple[str, str, str, str | None, str, str, str, str]: """ base_path = os.environ.get('BENCH_RESULTS', None) csp = os.environ.get('CSP', None) + attestation_variant = os.environ.get('ATTESTATION_VARIANT', None) out_dir = os.environ.get('BDIR', None) - if not base_path or not csp or not out_dir: + if not base_path or not csp or not out_dir or not attestation_variant: raise TypeError( - 'ENV variables BENCH_RESULTS, CSP, BDIR are required.') + 'ENV variables BENCH_RESULTS, CSP, BDIR, ATTESTATION_VARIANT are required.') ext_provider_name = os.environ.get('EXT_NAME', None) commit_hash = os.environ.get('GITHUB_SHA', 'N/A') commit_ref = os.environ.get('GITHUB_REF_NAME', 'N/A') actor = os.environ.get('GITHUB_ACTOR', 'N/A') workflow = os.environ.get('GITHUB_WORKFLOW', 'N/A') - return base_path, csp, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow + return base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow class BenchmarkParser: - def __init__(self, base_path, csp, out_dir, ext_provider_name=None, commit_hash="N/A", commit_ref="N/A", actor="N/A", workflow="N/A"): + def __init__(self, base_path, csp, attestation_variant, out_dir, ext_provider_name=None, commit_hash="N/A", commit_ref="N/A", actor="N/A", workflow="N/A"): self.base_path = base_path self.csp = csp + self.attestation_variant = attestation_variant self.out_dir = out_dir self.ext_provider_name = ext_provider_name if not self.ext_provider_name: - self.ext_provider_name = f'constellation-{csp}' + self.ext_provider_name = f'constellation-{attestation_variant}' self.commit_hash = commit_hash self.commit_ref = commit_ref self.actor = actor @@ -88,6 +90,7 @@ class BenchmarkParser: }, '@timestamp': str(timestamp), 'provider': self.ext_provider_name, + 'attestationVariant': self.attestation_variant, 'fio': {}, 'knb': {}} @@ -101,8 +104,8 @@ class BenchmarkParser: def main(): - base_path, csp, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow = configure() - p = BenchmarkParser(base_path, csp, out_dir, ext_provider_name, + base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow = configure() + p = BenchmarkParser(base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow) p.parse() diff --git a/.github/actions/e2e_benchmark/evaluate/requirements.txt b/.github/actions/e2e_benchmark/evaluate/requirements.txt index ad4f5ea1d..c2208b0c2 100644 --- a/.github/actions/e2e_benchmark/evaluate/requirements.txt +++ b/.github/actions/e2e_benchmark/evaluate/requirements.txt @@ -1,3 +1,3 @@ -numpy ==1.24.3 -matplotlib ==3.7.1 -Pillow ==10.0.1 \ No newline at end of file +numpy ==2.3.0 +matplotlib ==3.10.3 +Pillow ==11.3.0 \ No newline at end of file diff --git a/.github/actions/e2e_benchmark/fio.ini b/.github/actions/e2e_benchmark/fio.ini index c956bc7b4..33960341a 100644 --- a/.github/actions/e2e_benchmark/fio.ini +++ b/.github/actions/e2e_benchmark/fio.ini @@ -7,7 +7,7 @@ size=10Gi time_based=1 group_reporting thread -cpus_allowed=1 +cpus_allowed=0 [read_iops] diff --git a/.github/actions/e2e_cleanup_timeframe/action.yml b/.github/actions/e2e_cleanup_timeframe/action.yml new file mode 100644 index 000000000..c77be2431 --- /dev/null +++ b/.github/actions/e2e_cleanup_timeframe/action.yml @@ -0,0 +1,62 @@ +name: E2E cleanup over timeframe +description: Clean up old terraform resources of E2E tests + +inputs: + ghToken: + description: 'The github token that is used with the github CLI.' + required: true + encryptionSecret: + description: 'The secret to use for decrypting the artifacts.' + required: true + azure_credentials: + description: "Credentials authorized to create Constellation on Azure." + required: true + openStackCloudsYaml: + description: "The contents of ~/.config/openstack/clouds.yaml" + required: false + stackitUat: + description: "The UAT for STACKIT" + required: false + +runs: + using: "composite" + steps: + - name: Authenticate AWS + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + with: + role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EDestroy + aws-region: eu-central-1 + + - name: Authenticate Azure + uses: ./.github/actions/login_azure + with: + azure_credentials: ${{ inputs.azure_credentials }} + + - name: Authenticate GCP + uses: ./.github/actions/login_gcp + with: + service_account: "destroy-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Login to OpenStack + uses: ./.github/actions/login_openstack + with: + clouds_yaml: ${{ inputs.openStackCloudsYaml }} + + - name: Login to STACKIT + uses: ./.github/actions/login_stackit + with: + serviceAccountToken: ${{ inputs.stackitUat }} + + - name: Install tools + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: | + _7zz + terraform + + - name: Run cleanup + run: ./.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh + shell: bash + env: + GH_TOKEN: ${{ inputs.ghToken }} + ENCRYPTION_SECRET: ${{ inputs.encryptionSecret }} diff --git a/.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh b/.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh new file mode 100755 index 000000000..0796ae1a0 --- /dev/null +++ b/.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# get_e2e_test_ids_on_date gets all workflow IDs of workflows that contain "e2e" on a specific date. +function get_e2e_test_ids_on_date { + ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | (contains("e2e") or contains("Release")) and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)" + echo "${ids}" +} + +# download_tfstate_artifact downloads all artifacts matching the pattern terraform-state-* from a given workflow ID. +function download_tfstate_artifact { + gh run download "$1" -p "terraform-state-*" -R edgelesssys/constellation > /dev/null +} + +# delete_terraform_resources runs terraform destroy on the given folder. +function delete_terraform_resources { + delete_err=0 + if pushd "${1}/${2}"; then + # Workaround for cleaning up Azure resources + # We include a data source that is only used to generate output + # If this data source is deleted before we call terraform destroy, + # terraform will first try to evaluate the data source and fail, + # causing the destroy to fail as well. + sed -i '/data "azurerm_user_assigned_identity" "uaid" {/,/}/d' main.tf + sed -i '/output "user_assigned_identity_client_id" {/,/}/d' outputs.tf + + terraform init > /dev/null || delete_err=1 # first, install plugins + terraform destroy -auto-approve || delete_err=1 + popd || exit 1 + fi + return "${delete_err}" +} + +# check if the password for artifact decryption was given +if [[ -z ${ENCRYPTION_SECRET} ]]; then + echo "ENCRYPTION_SECRET is not set. Please set an environment variable with that secret." + exit 1 +fi + +artifact_pwd=${ENCRYPTION_SECRET} + +shopt -s nullglob + +start_date=$(date "+%Y-%m-%d") +end_date=$(date --date "-4 day" "+%Y-%m-%d") +dates_to_clean=() + +# get all dates of the last week +while [[ ${end_date} != "${start_date}" ]]; do + dates_to_clean+=("${end_date}") + end_date=$(date --date "${end_date} +1 day" "+%Y-%m-%d") +done + +echo "[*] retrieving run IDs for cleanup" +database_ids=() +for d in "${dates_to_clean[@]}"; do + echo " retrieving run IDs from $d" + mapfile -td " " tmp < <(get_e2e_test_ids_on_date "$d") + database_ids+=("${tmp[*]}") +done + +# cleanup database_ids +mapfile -t database_ids < <(echo "${database_ids[@]}") +mapfile -td " " database_ids < <(echo "${database_ids[@]}") + +echo "[*] downloading terraform state artifacts" +for id in "${database_ids[@]}"; do + if [[ ${id} == *[^[:space:]]* ]]; then + echo " downloading from workflow ${id}" + download_tfstate_artifact "${id}" + fi +done + +echo "[*] extracting artifacts" +for directory in ./terraform-state-*; do + echo " extracting ${directory}" + + # extract and decrypt the artifact + 7zz x -t7z -p"${artifact_pwd}" -o"${directory}" "${directory}/archive.7z" > /dev/null || exit 1 +done + +# create terraform caching directory +mkdir "${HOME}/tf_plugin_cache" +export TF_PLUGIN_CACHE_DIR="${HOME}/tf_plugin_cache" +echo "[*] created terraform cache directory ${TF_PLUGIN_CACHE_DIR}" + +echo "[*] deleting resources" +error_occurred=0 +for directory in ./terraform-state-*; do + echo " deleting resources in ${directory}" + if ! delete_terraform_resources "${directory}" "constellation-terraform"; then + echo "[!] deleting resources failed" + error_occurred=1 + fi + echo " deleting IAM configuration in ${directory}" + if ! delete_terraform_resources "${directory}" "constellation-iam-terraform"; then + echo "[!] deleting IAM resources failed" + error_occurred=1 + fi + echo " deleting directory ${directory}" + rm -rf "${directory}" +done + +if [[ ${error_occurred} -ne 0 ]]; then + echo "[!] Errors occurred during resource deletion." + exit 1 +fi + +exit 0 diff --git a/.github/actions/e2e_emergency_ssh/action.yml b/.github/actions/e2e_emergency_ssh/action.yml new file mode 100644 index 000000000..27b3e8b13 --- /dev/null +++ b/.github/actions/e2e_emergency_ssh/action.yml @@ -0,0 +1,70 @@ +name: Emergency ssh +description: "Verify that an emergency ssh connection can be established." + +inputs: + kubeconfig: + description: "The kubeconfig file for the cluster." + required: true + +runs: + using: "composite" + steps: + - name: Test emergency ssh + shell: bash + env: + KUBECONFIG: ${{ inputs.kubeconfig }} + run: | + set -euo pipefail + + # Activate emergency ssh access to the cluster + pushd ./constellation-terraform + echo "emergency_ssh = true" >> terraform.tfvars + terraform apply -auto-approve + lb="$(terraform output -raw loadbalancer_address)" + popd + + lb_ip="$(gethostip $lb | awk '{print $2}')" + echo "Resolved ip of load balancer: $lb_ip" + + # write ssh config + cat > ssh_config <> $GITHUB_PATH export PATH="$PATH:$(pwd)" constellation version - # Do not spam license server from pipeline - sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' - name: Build Terraform provider binary if: inputs.clusterCreation == 'terraform' && inputs.cliVersion == '' @@ -214,7 +229,7 @@ runs: - name: Login to AWS (IAM role) if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM aws-region: eu-central-1 @@ -227,30 +242,55 @@ runs: with: azure_credentials: ${{ inputs.azureIAMCreateCredentials }} + - name: Login to OpenStack + if: inputs.cloudProvider == 'stackit' + uses: ./.github/actions/login_openstack + with: + clouds_yaml: ${{inputs.openStackCloudsYaml }} + + - name: Login to STACKIT + if: inputs.cloudProvider == 'stackit' + uses: ./.github/actions/login_stackit + with: + serviceAccountToken: ${{ inputs.stackitUat }} + - name: Create prefix id: create-prefix shell: bash run: | uuid=$(uuidgen | tr "[:upper:]" "[:lower:]") uuid=${uuid%%-*} + + # GCP has a 6 character limit the additional uuid prefix since the full prefix length has a maximum of 24 + if [[ ${{ inputs.cloudProvider }} == 'gcp' ]]; then + uuid=${uuid:0:6} + fi + echo "uuid=${uuid}" | tee -a $GITHUB_OUTPUT echo "prefix=e2e-${{ github.run_id }}-${{ github.run_attempt }}-${uuid}" | tee -a $GITHUB_OUTPUT - name: Pick a random Azure region id: pick-az-region uses: ./.github/actions/pick_azure_region + with: + attestationVariant: ${{ inputs.attestationVariant }} - - name: Create IAM configuration + - name: Create Constellation config and IAM id: constellation-iam-create uses: ./.github/actions/constellation_iam_create with: cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} namePrefix: ${{ steps.create-prefix.outputs.prefix }} awsZone: ${{ inputs.regionZone || 'us-east-2c' }} + azureSubscriptionID: ${{ inputs.azureSubscriptionID }} azureRegion: ${{ inputs.regionZone || steps.pick-az-region.outputs.region }} gcpProjectID: ${{ inputs.gcpProject }} gcpZone: ${{ inputs.regionZone || 'europe-west3-b' }} + stackitZone: ${{ inputs.regionZone || 'eu01-2' }} + stackitProjectID: ${{ inputs.stackitProjectID }} kubernetesVersion: ${{ inputs.kubernetesVersion }} + additionalTags: "workflow=${{ github.run_id }}" - name: Login to GCP (Cluster service account) if: inputs.cloudProvider == 'gcp' @@ -260,7 +300,7 @@ runs: - name: Login to AWS (Cluster role) if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster aws-region: eu-central-1 @@ -278,6 +318,7 @@ runs: uses: ./.github/actions/constellation_create with: cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} workerNodesCount: ${{ inputs.workerNodesCount }} controlNodesCount: ${{ inputs.controlNodesCount }} machineType: ${{ inputs.machineType }} @@ -296,6 +337,7 @@ runs: clusterCreation: ${{ inputs.clusterCreation }} marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }} force: ${{ inputs.force }} + encryptionSecret: ${{ inputs.encryptionSecret }} - name: Deploy log- and metrics-collection (Kubernetes) id: deploy-logcollection @@ -307,6 +349,7 @@ runs: opensearchPwd: ${{ inputs.awsOpenSearchPwd }} test: ${{ inputs.test }} provider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} isDebugImage: ${{ inputs.isDebugImage }} kubernetesVersion: ${{ inputs.kubernetesVersion }} refStream: ${{ inputs.refStream }} @@ -319,7 +362,7 @@ runs: if: (inputs.test == 'nop') || (inputs.test == 'upgrade') shell: bash run: | - echo "::warning::This test has a nop payload. It doesn't run any tests." + echo "This test has a nop payload. It doesn't run any tests." echo "Sleeping for 30 seconds to allow logs to propagate to the log collection service." sleep 30 @@ -330,15 +373,26 @@ runs: sonobuoyTestSuiteCmd: "--mode quick" kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }} + encryptionSecret: ${{ inputs.encryptionSecret }} - name: Run sonobuoy full test if: inputs.test == 'sonobuoy full' uses: ./.github/actions/e2e_sonobuoy with: - # TODO(3u13r): Remove E2E_SKIP once AB#2174 is resolved - sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml' + # TODO: Remove E2E_SKIP once AB#2174 is resolved + sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol|Services should serve endpoints on same port and different protocols" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/102cd62a4091f80a795189f64ccc20738f931ef0/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/102cd62a4091f80a795189f64ccc20738f931ef0/cis-benchmarks/kube-bench-master-plugin.yaml' kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }} + encryptionSecret: ${{ inputs.encryptionSecret }} + + - name: Run sonobuoy conformance + if: inputs.test == 'sonobuoy conformance' + uses: ./.github/actions/e2e_sonobuoy + with: + sonobuoyTestSuiteCmd: "--plugin e2e --mode certified-conformance" + kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} + artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }} + encryptionSecret: ${{ inputs.encryptionSecret }} - name: Run autoscaling test if: inputs.test == 'autoscaling' @@ -351,22 +405,26 @@ runs: uses: ./.github/actions/e2e_lb with: kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} + cloudProvider: ${{ inputs.cloudProvider }} - name: Run Performance Benchmark if: inputs.test == 'perf-bench' uses: ./.github/actions/e2e_benchmark with: cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} awsOpenSearchDomain: ${{ inputs.awsOpenSearchDomain }} awsOpenSearchUsers: ${{ inputs.awsOpenSearchUsers }} awsOpenSearchPwd: ${{ inputs.awsOpenSearchPwd }} + encryptionSecret: ${{ inputs.encryptionSecret }} + artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }} - name: Run constellation verify test if: inputs.test == 'verify' uses: ./.github/actions/e2e_verify with: - cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} osImage: ${{ steps.constellation-create.outputs.osImageUsed }} kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} cosignPassword: ${{ inputs.cosignPassword }} @@ -384,6 +442,7 @@ runs: uses: ./.github/actions/e2e_malicious_join with: cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} githubToken: ${{ inputs.githubToken }} @@ -394,5 +453,10 @@ runs: kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} s3AccessKey: ${{ inputs.s3AccessKey }} s3SecretKey: ${{ inputs.s3SecretKey }} - buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }} githubToken: ${{ inputs.githubToken }} + + - name: Run emergency ssh test + if: inputs.test == 'emergency ssh' + uses: ./.github/actions/e2e_emergency_ssh + with: + kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} diff --git a/.github/actions/e2e_verify/action.yml b/.github/actions/e2e_verify/action.yml index ae402fa26..6803124f6 100644 --- a/.github/actions/e2e_verify/action.yml +++ b/.github/actions/e2e_verify/action.yml @@ -5,8 +5,8 @@ inputs: osImage: description: "The OS image used in the cluster." required: true - cloudProvider: - description: "The cloud provider used in the cluster." + attestationVariant: + description: "The attestation variant used in the cluster." required: true kubeconfig: description: "The kubeconfig file for the cluster." @@ -66,43 +66,46 @@ runs: forwarderPID=$! sleep 5 - # TODO(v2.15): Remove workaround since we don't need to support v2.13 anymore - if [[ ${{ inputs.cloudProvider }} == "azure" ]] || { [[ ${{ inputs.cloudProvider }} == "aws" ]] && ! constellation version | grep -q "v2.13."; }; then - echo "Extracting TCB versions for API update" - constellation verify --cluster-id "${clusterID}" --node-endpoint localhost:9090 -o json > "snp-report-${node}.json" - else - constellation verify --cluster-id "${clusterID}" --node-endpoint localhost:9090 - fi + case "${{ inputs.attestationVariant }}" + in + "azure-sev-snp"|"azure-tdx"|"aws-sev-snp"|"gcp-sev-snp") + echo "Extracting TCB versions for API update" + constellation verify --cluster-id "${clusterID}" --node-endpoint localhost:9090 -o json > "attestation-report-${node}.json" + ;; + *) + constellation verify --cluster-id "${clusterID}" --node-endpoint localhost:9090 + ;; + esac kill $forwarderPID done - name: Login to AWS if: github.ref_name == 'main' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline aws-region: eu-central-1 - name: Upload extracted TCBs - if: github.ref_name == 'main' && (inputs.cloudProvider == 'azure' || inputs.cloudProvider == 'aws') + if: github.ref_name == 'main' && (inputs.attestationVariant == 'azure-sev-snp' || inputs.attestationVariant == 'azure-tdx' || inputs.attestationVariant == 'aws-sev-snp' || inputs.attestationVariant == 'gcp-sev-snp') shell: bash env: COSIGN_PASSWORD: ${{ inputs.cosignPassword }} COSIGN_PRIVATE_KEY: ${{ inputs.cosignPrivateKey }} run: | - if [[ ${{ inputs.cloudProvider }} == "aws" ]] && constellation version | grep -q "v2.13."; then - echo "Skipping TCB upload for AWS on CLI v2.13" - exit 0 - fi + reports=attestation-report-*.json - reports=(snp-report-*.json) - if [ -z ${#reports[@]} ]; then - exit 1 - fi - - for file in "${reports[@]}"; do - path=$(realpath "${file}") - cat "${path}" - bazel run //internal/api/attestationconfigapi/cli -- upload ${{ inputs.cloudProvider }} snp-report "${path}" + # bazel run changes the working directory + # convert the relative paths to absolute paths to avoid issues + absolute_reports="" + for report in ${reports}; do + absolute_reports="${absolute_reports} $(realpath "${report}")" done + + report=$(bazel run //internal/api/attestationconfigapi/cli -- compare ${{ inputs.attestationVariant }} ${absolute_reports}) + + path=$(realpath "${report}") + cat "${path}" + + bazel run //internal/api/attestationconfigapi/cli -- upload ${{ inputs.attestationVariant }} attestation-report "${path}" diff --git a/.github/actions/find_latest_image/action.yml b/.github/actions/find_latest_image/action.yml index 174b1807e..2495df405 100644 --- a/.github/actions/find_latest_image/action.yml +++ b/.github/actions/find_latest_image/action.yml @@ -26,23 +26,25 @@ runs: steps: - name: Checkout head if: inputs.imageVersion == '' && inputs.git-ref == 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.imageVersion == '' && inputs.git-ref != 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git-ref }} - name: Login to AWS if: inputs.imageVersion == '' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead aws-region: eu-central-1 + - uses: ./.github/actions/setup_bazel_nix + - name: Find latest image id: find-latest-image if: inputs.imageVersion == '' diff --git a/.github/actions/gcpccm_vers_to_build/findvers.sh b/.github/actions/gcpccm_vers_to_build/findvers.sh index dd35c582a..1b148d4f4 100755 --- a/.github/actions/gcpccm_vers_to_build/findvers.sh +++ b/.github/actions/gcpccm_vers_to_build/findvers.sh @@ -82,4 +82,4 @@ for major in "${allMajorVersions[@]}"; do done # Print one elem per line | quote elems | create array | remove empty elems and print compact. -printf '%s' "${versionsToBuild[@]}" | jq -R | jq -s | jq -c 'map(select(length > 0))' +printf '%s\n' "${versionsToBuild[@]}" | jq -R | jq -s | jq -c 'map(select(length > 0))' diff --git a/.github/actions/login_azure/action.yml b/.github/actions/login_azure/action.yml index bdbfede79..50ab87771 100644 --- a/.github/actions/login_azure/action.yml +++ b/.github/actions/login_azure/action.yml @@ -10,6 +10,6 @@ runs: # As described at: # https://github.com/Azure/login#configure-deployment-credentials - name: Login to Azure - uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.7 + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 with: creds: ${{ inputs.azure_credentials }} diff --git a/.github/actions/login_gcp/action.yml b/.github/actions/login_gcp/action.yml index 4a05a03b7..c32249c73 100644 --- a/.github/actions/login_gcp/action.yml +++ b/.github/actions/login_gcp/action.yml @@ -19,14 +19,12 @@ runs: echo "GCP_PROJECT=" >> "$GITHUB_ENV" echo "GOOGLE_CLOUD_PROJECT=" >> "$GITHUB_ENV" - # As described at: - # https://github.com/google-github-actions/setup-gcloud#service-account-key-json - name: Authorize GCP access - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1 + uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 with: - workload_identity_provider: projects/796962942582/locations/global/workloadIdentityPools/constellation-ci-pool/providers/constellation-ci-provider + workload_identity_provider: projects/1052692473304/locations/global/workloadIdentityPools/constellation-ci-pool/providers/constellation-ci-provider service_account: ${{ inputs.service_account }} # Even if preinstalled in Github Actions runner image, this setup does some magic authentication required for gsutil. - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 diff --git a/.github/actions/login_openstack/action.yml b/.github/actions/login_openstack/action.yml new file mode 100644 index 000000000..f009555b9 --- /dev/null +++ b/.github/actions/login_openstack/action.yml @@ -0,0 +1,16 @@ +name: OpenStack login +description: "Login to OpenStack" +inputs: + clouds_yaml: + description: "Credentials authorized to create Constellation on OpenStack." + required: true +runs: + using: "composite" + steps: + - name: Login to OpenStack + env: + CLOUDS_YAML: ${{ inputs.clouds_yaml }} + shell: bash + run: | + mkdir -p ~/.config/openstack + echo "${CLOUDS_YAML}" > ~/.config/openstack/clouds.yaml diff --git a/.github/actions/login_stackit/action.yml b/.github/actions/login_stackit/action.yml new file mode 100644 index 000000000..a7ff58425 --- /dev/null +++ b/.github/actions/login_stackit/action.yml @@ -0,0 +1,16 @@ +name: STACKIT login +description: "Login to STACKIT" +inputs: + serviceAccountToken: + description: "Credentials authorized to create Constellation on STACKIT." + required: true +runs: + using: "composite" + steps: + - name: Login to STACKIT + env: + UAT: ${{ inputs.serviceAccountToken }} + shell: bash + run: | + mkdir -p ~/.stackit + echo "${UAT}" > ~/.stackit/credentials.json diff --git a/.github/actions/notify_e2e_failure/action.yml b/.github/actions/notify_e2e_failure/action.yml index fa8b6cb51..56463a23f 100644 --- a/.github/actions/notify_e2e_failure/action.yml +++ b/.github/actions/notify_e2e_failure/action.yml @@ -11,6 +11,9 @@ inputs: provider: description: "CSP" required: true + attestationVariant: + description: "Attestation variant" + required: false refStream: description: "RefStream of the run" required: false @@ -18,8 +21,8 @@ inputs: description: "Kubernetes version" required: false clusterCreation: - description: "How the infrastructure for the e2e test was created. One of [cli, self-managed, terraform]." - default: "false" + description: "How the infrastructure for the e2e test was created. One of [cli, terraform]." + required: false runs: using: "composite" @@ -39,14 +42,47 @@ runs: run: | # TODO(katexochen): add job number when possible jobURL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - # TODO(msanft): Add Self-managed param once logcollection is fixed. - opensearchURL="https://search-e2e-logs-y46renozy42lcojbvrt3qq7csm.eu-central-1.es.amazonaws.com/_dashboards/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-7d,to:now))&_a=(columns:!(metadata.name,systemd.unit,kubernetes.pod_name,message),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.github.e2e-test-provider,negate:!f,params:(query:${{ inputs.provider }}),type:phrase),query:(match_phrase:(metadata.github.e2e-test-provider:${{ inputs.provider }}))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.github.run-id,negate:!f,params:(query:${{ github.run_id }}),type:phrase),query:(match_phrase:(metadata.github.run-id:${{ github.run_id }}))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.github.ref-stream.keyword,negate:!f,params:(query:'${{ inputs.refStream }}'),type:phrase),query:(match_phrase:(metadata.github.ref-stream.keyword:'${{ inputs.refStream }}'))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.github.kubernetes-version.keyword,negate:!f,params:(query:'${{ inputs.kubernetesVersion }}'),type:phrase),query:(match_phrase:(metadata.github.kubernetes-version.keyword:'${{ inputs.kubernetesVersion }}'))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',key:metadata.github.e2e-test-payload,negate:!f,params:(query:'${{ inputs.test }}'),type:phrase),query:(match_phrase:(metadata.github.e2e-test-payload:'${{ inputs.test }}')))),index:'74517cf0-6442-11ed-acf1-47dda8fdfbbb',interval:auto,query:(language:kuery,query:''),sort:!())" + + # OpenSearch instance details + instance=search-e2e-logs-y46renozy42lcojbvrt3qq7csm + region=eu-central-1 + + # UUID of index "logs-*" + a="(metadata:(indexPattern:'9004ee20-77cc-11ee-b137-27c60b9ad4a4',view:discover))" + + # Default window: last 7 days + g='(time:(from:now-7d,to:now))' + + # Query construction + # Omit empty fields since OpenSearch will otherwise only display results where the field is empty + queryGen() { + key=$1 + val=$2 + if [[ -n "${val}" ]]; then + printf "(query:(match_phrase:(%s:'%s')))," "${key}" "${val}" + fi + } + + e2eTestPayload=$(echo "${{ inputs.test }}" | jq -R -r @uri) + + q=$(echo "(filters:!( + $(queryGen cloud.provider "${{ inputs.provider }}") + $(queryGen metadata.github.ref-stream "${{ inputs.refStream }}") + $(queryGen metadata.github.kubernetes-version "${{ inputs.kubernetesVersion }}") + $(queryGen metadata.github.attestation-variant "${{ inputs.attestationVariant }}") + $(queryGen metadata.github.cluster-creation "${{ inputs.clusterCreation }}") + $(queryGen metadata.github.e2e-test-payload "${e2eTestPayload}") + (query:(match_phrase:(metadata.github.run-id:${{ github.run_id }}))) + ))" | tr -d "\t\n ") + + # URL construction + opensearchURL="https://${instance}.${region}.es.amazonaws.com/_dashboards/app/data-explorer/discover/#?_a=${a}&_q=${q}&_g=${g}" cat << EOF > header.md ## Metadata * [Job URL](${jobURL}) - * [OpenSearch URL](${opensearchURL// /%20}) + * [OpenSearch URL](${opensearchURL}) EOF @@ -66,6 +102,7 @@ runs: workflow: ${{ github.workflow }} kubernetesVersion: ${{ inputs.kubernetesVersion }} cloudProvider: ${{ inputs.provider }} + attestationVariant: ${{ inputs.attestationVariant }} clusterCreation: ${{ inputs.clusterCreation }} test: ${{ inputs.test }} refStream: ${{ inputs.refStream }} diff --git a/.github/actions/notify_stackit/action.yml b/.github/actions/notify_stackit/action.yml new file mode 100644 index 000000000..2e64fdac5 --- /dev/null +++ b/.github/actions/notify_stackit/action.yml @@ -0,0 +1,19 @@ +name: Notify STACKIT +description: "Notify STACKIT about test failure" +inputs: + slackToken: + description: "Slack access token." + required: true +runs: + using: "composite" + steps: + - name: Notify STACKIT + env: + SLACK_TOKEN: ${{ inputs.slackToken }} + shell: bash + run: | + curl -X POST \ + -H "Authorization: Bearer $SLACK_TOKEN" \ + -H "Content-type: application/json; charset=utf-8" \ + -d "{\"channel\":\"C0827BT59SM\",\"text\":\"E2E test failed: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID\"}" \ + https://slack.com/api/chat.postMessage diff --git a/.github/actions/notify_teams/README.md b/.github/actions/notify_teams/README.md new file mode 100644 index 000000000..5fb6d724b --- /dev/null +++ b/.github/actions/notify_teams/README.md @@ -0,0 +1,27 @@ +# notify Teams action + +This action is used to send a message to our Teams channel in case of a failure in the CI/CD pipeline. +The action will automatically choose an engineer to assign to the issue and tag them in the message. + +Engineers are identified by their GitHub username and bound to a Microsoft Teams ID in `.attachments[0].content.msteams.entities`. +To add a new engineer, add a new entry to the entity list in the format: + +```json +{ + "type": "mention", + "text": "${github_username}", + "mentioned": { + "id": "${msteams_id}", + "name": "${name}" + } +} +``` + +Where `${github_username}` is the GitHub username of the engineer, `${msteams_id}` is the Microsoft Teams ID of the engineer, and `${name}` is the name of the engineer. +To find the Microsoft Teams ID use the following command: + +```bash +az ad user show --id ${email} --query id +``` + +Where `${email}` is the email address of the engineer. diff --git a/.github/actions/notify_teams/action.yml b/.github/actions/notify_teams/action.yml index 956a3a889..e94a266a9 100644 --- a/.github/actions/notify_teams/action.yml +++ b/.github/actions/notify_teams/action.yml @@ -25,7 +25,7 @@ runs: continue-on-error: true shell: bash run: | - cp .github/teams_payload_template.json teams_payload.json + cp .github/actions/notify_teams/teams_payload_template.json teams_payload.json # Add workflow name to the notification yq -oj -iP '.attachments[0].content.body[0].columns[1].items[0].text = "${{ inputs.title }}"' teams_payload.json diff --git a/.github/teams_payload_template.json b/.github/actions/notify_teams/teams_payload_template.json similarity index 76% rename from .github/teams_payload_template.json rename to .github/actions/notify_teams/teams_payload_template.json index 145d6a28c..0354bc07b 100644 --- a/.github/teams_payload_template.json +++ b/.github/actions/notify_teams/teams_payload_template.json @@ -1,5 +1,5 @@ { - "type": "message", + "type": "AdaptiveCard", "attachments": [ { "contentType": "application/vnd.microsoft.card.adaptive", @@ -11,14 +11,6 @@ "msteams": { "width": "Full", "entities": [ - { - "type": "mention", - "text": "elchead", - "mentioned": { - "id": "3931943b-8d4b-4300-ac7e-bbb06c4da27f", - "name": "Adrian Stobbe" - } - }, { "type": "mention", "text": "msanft", @@ -27,14 +19,6 @@ "name": "Moritz Sanft" } }, - { - "type": "mention", - "text": "3u13r", - "mentioned": { - "id": "26869b29-b0d6-48f8-a9ed-7a6374410a53", - "name": "Leonard Cohnen" - } - }, { "type": "mention", "text": "daniel-weisse", @@ -53,18 +37,10 @@ }, { "type": "mention", - "text": "derpsteb", + "text": "burgerdev", "mentioned": { - "id": "a9a34611-9a38-4c00-a8a2-f87d94c2bf7d", - "name": "Otto Bittner" - } - }, - { - "type": "mention", - "text": "malt3", - "mentioned": { - "id": "3012fe21-cff7-499d-88cf-48cf12f2e90c", - "name": "Malte Poll" + "id": "c9efc581-58ca-4da6-93ce-79f69f89deeb", + "name": "Markus Rudy" } } ] diff --git a/.github/actions/pick_assignee/action.yml b/.github/actions/pick_assignee/action.yml index 5a059e829..ed9607e77 100644 --- a/.github/actions/pick_assignee/action.yml +++ b/.github/actions/pick_assignee/action.yml @@ -14,12 +14,9 @@ runs: shell: bash run: | possibleAssignees=( - "elchead" - "malt3" - "3u13r" "daniel-weisse" - "derpsteb" "msanft" + "burgerdev" ) assignee=${possibleAssignees[$RANDOM % ${#possibleAssignees[@]}]} echo "assignee=$assignee" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/actions/pick_azure_region/action.yml b/.github/actions/pick_azure_region/action.yml index a37ddd671..8fb7bb538 100644 --- a/.github/actions/pick_azure_region/action.yml +++ b/.github/actions/pick_azure_region/action.yml @@ -1,6 +1,11 @@ name: Pick an Azure region description: "Pick an Azure region" +inputs: + attestationVariant: + description: "Attestation variant to use. Not all regions support all variants." + required: true + outputs: region: description: "One of the supported Azure regions" @@ -13,12 +18,25 @@ runs: id: pick-region shell: bash run: | - possibleRegions=( + possibleRegionsSNP=( "westus" "eastus" "northeurope" "westeurope" "southeastasia" ) + possibleRegionsTDX=( + "centralus" + "eastus2" + "northeurope" + "westeurope" + ) + + if [[ "${{ inputs.attestationVariant }}" == "azure-tdx" ]]; then + possibleRegions=("${possibleRegionsTDX[@]}") + else + possibleRegions=("${possibleRegionsSNP[@]}") + fi + region=${possibleRegions[$RANDOM % ${#possibleRegions[@]}]} echo "region=$region" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/actions/publish_helmchart/action.yml b/.github/actions/publish_helmchart/action.yml index 8e527cf95..3c26fbad7 100644 --- a/.github/actions/publish_helmchart/action.yml +++ b/.github/actions/publish_helmchart/action.yml @@ -13,7 +13,7 @@ runs: using: "composite" steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: edgelesssys/helm ref: main @@ -29,7 +29,7 @@ runs: echo version=$(yq eval ".version" ${{ inputs.chartPath }}/Chart.yaml) | tee -a $GITHUB_OUTPUT - name: Create pull request - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: path: helm branch: "release/s3proxy/${{ steps.update-chart-version.outputs.version }}" diff --git a/.github/actions/select_image/action.yml b/.github/actions/select_image/action.yml index 08a05554c..6af36480d 100644 --- a/.github/actions/select_image/action.yml +++ b/.github/actions/select_image/action.yml @@ -3,22 +3,22 @@ description: Resolve string presets and shortpaths to shortpaths only inputs: osImage: - description: "Shortpath or main-debug or release-stable" + description: "Shortpath, main-debug, main-nightly, or release-stable" required: true outputs: osImage: - description: "Shortpath of for input string, original input if that was already a shortpath" + description: "Shortpath of input string, original input if that was already a shortpath" value: ${{ steps.set-output.outputs.osImage }} isDebugImage: - description: "Input represents a debug image or not" + description: "Input is a debug image or not" value: ${{ steps.set-output.outputs.isDebugImage }} runs: using: "composite" steps: - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead aws-region: eu-central-1 @@ -27,7 +27,7 @@ runs: id: input-is-preset shell: bash run: | - if [[ "${{ inputs.osImage }}" == "ref/main/stream/debug/?" || "${{ inputs.osImage }}" == "ref/release/stream/stable/?" ]]; then + if [[ "${{ inputs.osImage }}" == "ref/main/stream/debug/?" || "${{ inputs.osImage }}" == "ref/main/stream/nightly/?" || "${{ inputs.osImage }}" == "ref/release/stream/stable/?" ]]; then echo "result=true" | tee -a "$GITHUB_OUTPUT" else echo "result=false" | tee -a "$GITHUB_OUTPUT" @@ -43,6 +43,10 @@ runs: echo "ref=$(echo $REFSTREAM | cut -d/ -f2)" | tee -a "$GITHUB_OUTPUT" echo "stream=$(echo $REFSTREAM | cut -d/ -f4)" | tee -a "$GITHUB_OUTPUT" + - name: Setup Bazel & Nix + if: steps.input-is-preset.outputs.result == 'true' + uses: ./.github/actions/setup_bazel_nix + - name: Find latest image if: steps.input-is-preset.outputs.result == 'true' id: find-latest-image diff --git a/.github/actions/self_managed_create/action.yml b/.github/actions/self_managed_create/action.yml deleted file mode 100644 index 354f7220c..000000000 --- a/.github/actions/self_managed_create/action.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: Self-managed infrastructure creation -description: "Create the required infrastructure for a Constellation cluster manually." - -inputs: - cloudProvider: - description: "The cloud provider the test runs on." - required: true - -runs: - using: "composite" - steps: - - name: Copy Terraform configuration and Constellation config - shell: bash - working-directory: - run: | - cp -r ${{ github.workspace }}/terraform/infrastructure/${{ inputs.cloudProvider }} ${{ github.workspace }}/e2e-infra - cp ${{ github.workspace }}/constellation-conf.yaml ${{ github.workspace }}/e2e-infra - - - name: Get CSP image reference - id: get_image - shell: bash - working-directory: ${{ github.workspace }}/e2e-infra - run: | - echo "image_ref=$(bazel run //hack/image-fetch:image-fetch)" >> $GITHUB_OUTPUT - - - name: Write Terraform variables - shell: bash - working-directory: ${{ github.workspace }}/e2e-infra - run: | - echo "name = \"$(yq '.name' constellation-conf.yaml)\"" >> terraform.tfvars - echo "debug = $(yq '.debugCluster' constellation-conf.yaml)" >> terraform.tfvars - echo "custom_endpoint = \"$(yq '.customEndpoint' constellation-conf.yaml)\"" >> terraform.tfvars - echo "image_id = \"${{ steps.get_image.outputs.image_ref }}\"" >> terraform.tfvars - echo "node_groups = { - control_plane_default = { - role = \"$(yq '.nodeGroups.control_plane_default.role' constellation-conf.yaml)\" - zone = \"$(yq '.nodeGroups.control_plane_default.zone' constellation-conf.yaml)\" - instance_type = \"$(yq '.nodeGroups.control_plane_default.instanceType' constellation-conf.yaml)\" - disk_size = \"$(yq '.nodeGroups.control_plane_default.stateDiskSizeGB' constellation-conf.yaml)\" - disk_type = \"$(yq '.nodeGroups.control_plane_default.stateDiskType' constellation-conf.yaml)\" - initial_count = \"$(yq '.nodeGroups.control_plane_default.initialCount' constellation-conf.yaml)\" - } - worker_default = { - role = \"$(yq '.nodeGroups.worker_default.role' constellation-conf.yaml)\" - zone = \"$(yq '.nodeGroups.worker_default.zone' constellation-conf.yaml)\" - instance_type = \"$(yq '.nodeGroups.worker_default.instanceType' constellation-conf.yaml)\" - disk_size = \"$(yq '.nodeGroups.worker_default.stateDiskSizeGB' constellation-conf.yaml)\" - disk_type = \"$(yq '.nodeGroups.worker_default.stateDiskType' constellation-conf.yaml)\" - initial_count = \"$(yq '.nodeGroups.worker_default.initialCount' constellation-conf.yaml)\" - } - }" >> terraform.tfvars - if [[ "${{ inputs.cloudProvider }}" == 'aws' ]]; then - echo "iam_instance_profile_name_control_plane = \"$(yq '.provider.aws.iamProfileControlPlane' constellation-conf.yaml)\"" >> terraform.tfvars - echo "iam_instance_profile_name_worker_nodes = \"$(yq '.provider.aws.iamProfileWorkerNodes' constellation-conf.yaml)\"" >> terraform.tfvars - echo "region = \"$(yq '.provider.aws.region' constellation-conf.yaml)\"" >> terraform.tfvars - echo "zone = \"$(yq '.provider.aws.zone' constellation-conf.yaml)\"" >> terraform.tfvars - echo "enable_snp = $(yq '.attestation | has("awsSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars - elif [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then - echo "location = \"$(yq '.provider.azure.location' constellation-conf.yaml)\"" >> terraform.tfvars - echo "create_maa = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars - echo "confidential_vm = $(yq '.attestation | has("azureSEVSNP")' constellation-conf.yaml)" >> terraform.tfvars - echo "secure_boot = $(yq '.provider.azure.secureBoot' constellation-conf.yaml)" >> terraform.tfvars - echo "resource_group = \"$(yq '.provider.azure.resourceGroup' constellation-conf.yaml)\"" >> terraform.tfvars - echo "user_assigned_identity = \"$(yq '.provider.azure.userAssignedIdentity' constellation-conf.yaml)\"" >> terraform.tfvars - elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then - echo "project = \"$(yq '.provider.gcp.project' constellation-conf.yaml)\"" >> terraform.tfvars - echo "region = \"$(yq '.provider.gcp.region' constellation-conf.yaml)\"" >> terraform.tfvars - echo "zone = \"$(yq '.provider.gcp.zone' constellation-conf.yaml)\"" >> terraform.tfvars - fi - terraform fmt terraform.tfvars - echo "Using Terraform variables:" - cat terraform.tfvars - - - name: Apply Terraform configuration - shell: bash - working-directory: ${{ github.workspace }}/e2e-infra - run: | - terraform init - terraform apply -auto-approve - - - name: Patch MAA Policy - shell: bash - working-directory: ${{ github.workspace }}/e2e-infra - if: inputs.cloudProvider == 'azure' - run: | - constellation maa-patch $(terraform output attestation_url | jq -r) - - - name: Write outputs to state file - shell: bash - working-directory: ${{ github.workspace }}/e2e-infra - run: | - yq eval '.version ="v1"' --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.initSecret =\"$(terraform output init_secret | jq -r | tr -d '\n' | hexdump -ve '/1 "%02x"' && echo '')\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.clusterEndpoint =\"$(terraform output out_of_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.inClusterEndpoint =\"$(terraform output in_cluster_endpoint | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.ipCidrNode =\"$(terraform output ip_cidr_node | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.uid =\"$(terraform output uid | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.name =\"$(terraform output name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.apiServerCertSANs =$(terraform output -json api_server_cert_sans)" --inplace ${{ github.workspace }}/constellation-state.yaml - if [[ "${{ inputs.cloudProvider }}" == 'azure' ]]; then - yq eval ".infrastructure.azure.resourceGroup =\"$(terraform output resource_group | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.azure.subscriptionID =\"$(terraform output subscription_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.azure.networkSecurityGroupName =\"$(terraform output network_security_group_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.azure.loadBalancerName =\"$(terraform output loadbalancer_name | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.azure.userAssignedIdentity =\"$(terraform output user_assigned_identity_client_id | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.azure.attestationURL =\"$(terraform output attestation_url | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - elif [[ "${{ inputs.cloudProvider }}" == 'gcp' ]]; then - yq eval ".infrastructure.gcp.projectID =\"$(terraform output project | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - yq eval ".infrastructure.gcp.ipCidrPod =\"$(terraform output ip_cidr_pod | jq -r)\"" --inplace ${{ github.workspace }}/constellation-state.yaml - fi diff --git a/.github/actions/setup_bazel_nix/action.yml b/.github/actions/setup_bazel_nix/action.yml index 7ce0202fe..b560ac8f6 100644 --- a/.github/actions/setup_bazel_nix/action.yml +++ b/.github/actions/setup_bazel_nix/action.yml @@ -3,12 +3,9 @@ description: Setup Bazel and Nix for CI builds and tests inputs: useCache: - description: "Cache Bazel artifacts. Use 'true' to enable with rw, 'readonly' to download, 'rbe' to enable with remote execution, 'log' to disable cache but upload logs, and 'false' to disable." + description: "Cache Bazel artifacts. Use 'rbe' to enable with remote execution, and 'false' to disable." default: "false" required: true - buildBuddyApiKey: - description: "BuildBuddy API key for caching Bazel artifacts" - required: false rbePlatform: description: "RBE platform to use. If empty, RBE will not be used." required: false @@ -25,12 +22,8 @@ runs: shell: bash run: | echo "::group::Check inputs" - if [[ "${{ inputs.useCache }}" != "true" && "${{ inputs.useCache }}" != "readonly" && "${{ inputs.useCache }}" != "rbe" && "${{ inputs.useCache }}" != "logs" && "${{ inputs.useCache }}" != "false" ]]; then - echo "Invalid value for 'useCache' input: '${{ inputs.useCache }}'. Must be 'true', 'readonly', or 'false'." - exit 1 - fi - if [[ "${{ inputs.useCache }}" == "true" || "${{ inputs.useCache }}" == "readonly" || "${{ inputs.useCache }}" == "logs" ]] && [[ -z "${{ inputs.buildBuddyApiKey }}" ]]; then - echo "BuildBuddy API key is required when cache is enabled." + if [[ "${{ inputs.useCache }}" != "rbe" && "${{ inputs.useCache }}" != "false" ]]; then + echo "Invalid value for 'useCache' input: '${{ inputs.useCache }}'. Must be 'rbe', or 'false'." exit 1 fi if [[ "${{ inputs.useCache }}" == "rbe" && -z "${{ inputs.rbePlatform }}" ]]; then @@ -82,8 +75,14 @@ runs: echo "$RUNNER_ARCH not supported" exit 1 fi + echo "nixVersion=$(cat "${{ github.workspace }}/.nixversion")" | tee -a "$GITHUB_OUTPUT" echo "::endgroup::" + - name: Install current Bash on macOS + shell: bash + if: runner.os == 'macOS' + run: brew install bash + - name: Prepare to install tools shell: bash run: | @@ -115,7 +114,9 @@ runs: - name: Install nix if: steps.check_inputs.outputs.nixPreinstalled == 'false' - uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23 + uses: cachix/install-nix-action@17fe5fb4a23ad6cbbe47d6b3f359611ad276644c # v31 + with: + install_url: "https://releases.nixos.org/nix/nix-${{ steps.check_inputs.outputs.nixVersion }}/install" - name: Set $USER if not set shell: bash @@ -177,57 +178,6 @@ runs: EOF echo "::endgroup::" - - name: Configure Bazel (rw) - if: inputs.useCache == 'true' || inputs.useCache == 'readonly' - shell: bash - env: - BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }} - WORKSPACE: ${{ github.workspace }} - run: | - echo "::group::Configure Bazel" - cat <> "${WORKSPACE}/.bazeloverwriterc" - common --bes_results_url=https://app.buildbuddy.io/invocation/ - common --bes_backend=grpcs://remote.buildbuddy.io - common --remote_cache=grpcs://remote.buildbuddy.io - common --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY} - cquery --bes_results_url= - cquery --bes_backend= - cquery --remote_cache= - query --bes_results_url= - query --bes_backend= - query --remote_cache= - EOF - echo "::endgroup::" - - - name: Configure Bazel (readonly) - if: inputs.useCache == 'readonly' - shell: bash - env: - WORKSPACE: ${{ github.workspace }} - run: | - echo "::group::Configure Bazel (readonly)" - echo "common --remote_upload_local_results=false" >> "${WORKSPACE}/.bazeloverwriterc" - echo "::endgroup::" - - - name: Configure Bazel (logs) - if: inputs.useCache == 'logs' - shell: bash - env: - BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }} - WORKSPACE: ${{ github.workspace }} - run: | - echo "::group::Configure Bazel" - cat <> "${WORKSPACE}/.bazeloverwriterc" - common --bes_results_url=https://app.buildbuddy.io/invocation/ - common --bes_backend=grpcs://remote.buildbuddy.io - common --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY} - cquery --bes_results_url= - cquery --bes_backend= - query --bes_results_url= - query --bes_backend= - EOF - echo "::endgroup::" - - name: Configure Bazel (rbe) if: inputs.useCache == 'rbe' shell: bash @@ -242,24 +192,6 @@ runs: common --repo_env=GOPROXY=http://goproxy:3000 EOF echo "::endgroup::" - - name: Configure Bazel (rbe logs) - if: inputs.useCache == 'rbe' && inputs.buildBuddyApiKey != '' - shell: bash - env: - BUILDBUDDY_ORG_API_KEY: ${{ inputs.buildBuddyApiKey }} - WORKSPACE: ${{ github.workspace }} - run: | - echo "::group::Configure Bazel" - cat <> "${WORKSPACE}/.bazeloverwriterc" - common --bes_results_url=https://app.buildbuddy.io/invocation/ - common --bes_backend=grpcs://remote.buildbuddy.io - common --remote_header=x-buildbuddy-api-key=${BUILDBUDDY_ORG_API_KEY} - cquery --bes_results_url= - cquery --bes_backend= - query --bes_results_url= - query --bes_backend= - EOF - echo "::endgroup::" - name: Disable disk cache on GitHub Actions runners if: startsWith(runner.name , 'GitHub Actions') @@ -276,6 +208,7 @@ runs: if: inputs.nixTools != '' shell: bash env: + NIXPKGS_ALLOW_UNFREE: 1 tools: ${{ inputs.nixTools }} repository: ${{ github.repository }} gitSha: ${{ github.sha }} @@ -288,7 +221,7 @@ runs: { tools, repository, rev }: let repoFlake = builtins.getFlake ("github:" + repository + "/" + rev); - nixpkgs = repoFlake.inputs.nixpkgsUnstable; + nixpkgs = repoFlake.inputs.nixpkgs; pkgs = import nixpkgs { system = builtins.currentSystem; }; toolPkgs = map (p: pkgs.${p}) tools; in diff --git a/.github/actions/terraform_apply/action.yml b/.github/actions/terraform_apply/action.yml index 0513ad79b..edf4fb26f 100644 --- a/.github/actions/terraform_apply/action.yml +++ b/.github/actions/terraform_apply/action.yml @@ -20,9 +20,18 @@ runs: "azureSEVSNP") attestationVariant="azure-sev-snp" ;; + "azureTDX") + attestationVariant="azure-tdx" + ;; "gcpSEVES") attestationVariant="gcp-sev-es" ;; + "gcpSEVSNP") + attestationVariant="gcp-sev-snp" + ;; + "qemuVTPM") + attestationVariant="qemu-vtpm" + ;; *) echo "Unknown attestation variant: $(yq '.attestation | keys | .[0]' constellation-conf.yaml)" exit 1 @@ -38,7 +47,7 @@ runs: } random = { source = "hashicorp/random" - version = "3.6.0" + version = "3.7.2" } } } @@ -83,6 +92,7 @@ runs: measurement_salt = random_bytes.measurement_salt.hex out_of_cluster_endpoint = "$(yq '.infrastructure.clusterEndpoint' constellation-state.yaml)" in_cluster_endpoint = "$(yq '.infrastructure.inClusterEndpoint' constellation-state.yaml)" + kubernetes_version = "$(yq '.kubernetesVersion' constellation-conf.yaml)" azure = { count = "$(yq '.provider | keys | .[0]' constellation-conf.yaml)" == "azure" ? 1 : 0 tenant_id = "$(yq '.provider.azure.tenant' constellation-conf.yaml)" @@ -99,6 +109,16 @@ runs: project_id = "$(yq '.infrastructure.gcp.projectID' constellation-state.yaml)" service_account_key = sensitive("$(cat $(yq '.provider.gcp.serviceAccountKeyPath' constellation-conf.yaml) | base64 -w0)") } + openstack = { + cloud = "stackit" + clouds_yaml_path = "~/.config/openstack/clouds.yaml" + floating_ip_pool_id = "970ace5c-458f-484a-a660-0903bcfd91ad" + deploy_yawol_load_balancer = true + yawol_image_id = "bcd6c13e-75d1-4c3f-bf0f-8f83580cc1be" + yawol_flavor_id = "3b11b27e-6c73-470d-b595-1d85b95a8cdf" + network_id = "$(yq '.infrastructure.networkID' constellation-state.yaml)" + subnet_id = "$(yq '.infrastructure.subnetID' constellation-state.yaml)" + } network_config = { ip_cidr_node = "$(yq '.infrastructure.ipCidrNode' constellation-state.yaml)" ip_cidr_service = "$(yq '.serviceCIDR' constellation-conf.yaml)" diff --git a/.github/actions/update_tfstate/action.yml b/.github/actions/update_tfstate/action.yml new file mode 100644 index 000000000..59aab2f04 --- /dev/null +++ b/.github/actions/update_tfstate/action.yml @@ -0,0 +1,64 @@ +name: Update TFState +description: "Update the terraform state artifact. We use this to either delete an artifact if the e2e test was cleaned up successfully or to update the artifact with the latest terraform state." + +inputs: + name: + description: "The name of the artifact that contains the tfstate." + required: true + runID: + description: "The ID of your current run (github.run_id)." + required: true + encryptionSecret: + description: "The encryption secret for the artifacts." + required: true + +runs: + using: "composite" + steps: + - name: Check if uploaded tfstate can be deleted + if: always() + shell: bash + run: | + if [[ ! -d constellation-terraform ]] && [[ ! -d constellation-iam-terraform ]]; then + echo "DELETE_TF_STATE=true" >> "$GITHUB_ENV" + else + echo "DELETE_TF_STATE=false" >> "$GITHUB_ENV" + fi + + - name: Delete tfstate artifact if necessary + if: always() && env.DELETE_TF_STATE == 'true' + uses: ./.github/actions/artifact_delete + with: + name: ${{ inputs.name }} + workflowID: ${{ inputs.runID }} + + - name: Prepare left over terraform state folders + if: always() && env.DELETE_TF_STATE == 'false' + shell: bash + run: | + rm -rf to-zip/* + mkdir -p to-zip + + to_upload="" + if [[ -d constellation-terraform ]]; then + cp -r constellation-terraform to-zip + rm -f to-zip/constellation-terraform/plan.zip + rm -rf to-zip/constellation-terraform/.terraform + to_upload+="to-zip/constellation-terraform" + fi + if [[ -d constellation-iam-terraform ]]; then + cp -r constellation-iam-terraform to-zip + rm -rf to-zip/constellation-iam-terraform/.terraform + to_upload+=" to-zip/constellation-iam-terraform" + fi + echo "TO_UPLOAD=$to_upload" >> "$GITHUB_ENV" + + - name: Update tfstate + if: always() && env.TO_UPLOAD != '' + uses: ./.github/actions/artifact_upload + with: + name: ${{ inputs.name }} + path: > + ${{ env.TO_UPLOAD }} + encryptionSecret: ${{ inputs.encryptionSecret }} + overwrite: true diff --git a/.github/actions/upload_terraform_module/action.yml b/.github/actions/upload_terraform_module/action.yml index 49c016e9f..140844fdd 100644 --- a/.github/actions/upload_terraform_module/action.yml +++ b/.github/actions/upload_terraform_module/action.yml @@ -15,7 +15,7 @@ runs: zip -r terraform-module.zip terraform-module - name: Upload artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: terraform-module path: terraform-module.zip @@ -23,4 +23,4 @@ runs: - name: Cleanup Terraform module dir shell: bash run: | - rm -r terraform-module terraform-module.zip + rm -rf terraform-module terraform-module.zip diff --git a/.github/actions/versionsapi/Dockerfile b/.github/actions/versionsapi/Dockerfile deleted file mode 100644 index 49869f5f5..000000000 --- a/.github/actions/versionsapi/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM golang:1.21.5@sha256:58e14a93348a3515c2becc54ebd35302128225169d166b7c6802451ab336c907 as builder - -# Download project root dependencies -WORKDIR /workspace -COPY go.mod go.mod -COPY go.sum go.sum -COPY operators/constellation-node-operator/api/go.mod ./operators/constellation-node-operator/api/go.mod -COPY operators/constellation-node-operator/api/go.sum ./operators/constellation-node-operator/api/go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -COPY . . - -# Build -WORKDIR /workspace/internal/api/versionsapi/cli -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o versionsapi . - -FROM scratch as release - -COPY --from=builder /workspace/internal/api/versionsapi/cli/versionsapi . - -CMD ["/notIntendedToBeExecuted"] diff --git a/.github/actions/versionsapi/action.yml b/.github/actions/versionsapi/action.yml index fac38adcb..817064334 100644 --- a/.github/actions/versionsapi/action.yml +++ b/.github/actions/versionsapi/action.yml @@ -52,18 +52,12 @@ outputs: runs: using: composite steps: - - name: Get versionsapi binary - shell: bash - run: | - containerID=$(docker create "ghcr.io/edgelesssys/constellation/versionsapi-ci-cli:latest") - docker cp ${containerID}:/versionsapi . - - name: Run versionsapi id: run shell: bash run: | out=$( - ./versionsapi \ + bazel run //internal/api/versionsapi/cli:cli -- \ ${{ inputs.command }} \ ${{ inputs.ref != '' && format('--ref="{0}"', inputs.ref) || '' }} \ ${{ inputs.stream != '' && format('--stream="{0}"', inputs.stream) || '' }} \ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 603f1fae3..92415ed65 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -26,6 +26,7 @@ Feel free to edit, complete or extend this list while the PR is open. ### Checklist +- [ ] Run the E2E tests that are relevant to this PR's changes - [ ] Update [docs](https://github.com/edgelesssys/constellation/tree/main/docs) - [ ] Add labels (e.g., for changelog category) - [ ] Is PR title adequate for changelog? diff --git a/.github/workflows/assign_reviewer.yml b/.github/workflows/assign_reviewer.yml new file mode 100644 index 000000000..ed87296d8 --- /dev/null +++ b/.github/workflows/assign_reviewer.yml @@ -0,0 +1,36 @@ +name: Assign Reviewer + +on: + pull_request: + types: + - opened + - reopened + - edited + - synchronize + - review_request_removed + - labeled + +permissions: + pull-requests: write + +jobs: + assign_reviewer: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'dependencies') && toJson(github.event.pull_request.requested_reviewers) == '[]' && github.event.pull_request.user.login == 'renovate[bot]' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Pick assignee + id: pick-assignee + uses: ./.github/actions/pick_assignee + - name: Assign reviewer + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + ASSIGNEE: ${{ steps.pick-assignee.outputs.assignee }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/edgelesssys/constellation/pulls/${PR}/requested_reviewers" \ + -f "reviewers[]=${ASSIGNEE}" diff --git a/.github/workflows/aws-snp-launchmeasurement.yml b/.github/workflows/aws-snp-launchmeasurement.yml index a5b303c03..d2483d71c 100644 --- a/.github/workflows/aws-snp-launchmeasurement.yml +++ b/.github/workflows/aws-snp-launchmeasurement.yml @@ -8,26 +8,20 @@ on: jobs: run: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} path: constellation - - name: Install necessary tools - run: | - sudo apt-get update - sudo apt-get install -y python3 python3-pip - sudo python3 -m pip install --user --require-hashes -r constellation/.github/workflows/aws-snp-launchmeasurements-requirements.txt - - name: Install Nix - uses: cachix/install-nix-action@6a9a9e84a173d90b3ffb42c5ddaf9ea033fad011 # v23 + uses: cachix/install-nix-action@17fe5fb4a23ad6cbbe47d6b3f359611ad276644c # v31 - name: Download Firmware release id: download-firmware - uses: robinraju/release-downloader@efa4cd07bd0195e6cc65e9e30c251b49ce4d3e51 # tag=v1.8 + uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1.12 with: repository: aws/uefi latest: true @@ -50,7 +44,7 @@ jobs: echo "ovmfPath=${ovmfPath}" | tee -a "$GITHUB_OUTPUT" popd || exit 1 - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: virtee/sev-snp-measure-go.git ref: e42b6f8991ed5a671d5d1e02a6b61f6373f9f8d8 diff --git a/.github/workflows/aws-snp-launchmeasurements-requirements.txt b/.github/workflows/aws-snp-launchmeasurements-requirements.txt deleted file mode 100644 index 08d9d1694..000000000 --- a/.github/workflows/aws-snp-launchmeasurements-requirements.txt +++ /dev/null @@ -1,109 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --generate-hashes --output-file=aws-snp-launchmeasurements-requirements.txt input.txt -# -cffi==1.15.1 \ - --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \ - --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \ - --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \ - --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \ - --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \ - --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \ - --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \ - --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \ - --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \ - --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \ - --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \ - --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \ - --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \ - --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \ - --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \ - --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \ - --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \ - --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \ - --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \ - --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \ - --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \ - --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \ - --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \ - --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \ - --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \ - --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \ - --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \ - --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \ - --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \ - --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \ - --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \ - --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \ - --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \ - --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \ - --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \ - --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \ - --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \ - --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \ - --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \ - --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \ - --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \ - --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \ - --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \ - --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \ - --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \ - --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \ - --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \ - --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \ - --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \ - --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \ - --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \ - --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \ - --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \ - --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \ - --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \ - --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \ - --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \ - --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \ - --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \ - --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \ - --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \ - --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \ - --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ - --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 - # via cryptography -cryptography==41.0.4 \ - --hash=sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67 \ - --hash=sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311 \ - --hash=sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8 \ - --hash=sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13 \ - --hash=sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143 \ - --hash=sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f \ - --hash=sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829 \ - --hash=sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd \ - --hash=sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397 \ - --hash=sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac \ - --hash=sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d \ - --hash=sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a \ - --hash=sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839 \ - --hash=sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e \ - --hash=sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6 \ - --hash=sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9 \ - --hash=sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860 \ - --hash=sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca \ - --hash=sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91 \ - --hash=sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d \ - --hash=sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714 \ - --hash=sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb \ - --hash=sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f - # via sev-snp-measure -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 - # via cffi -sev-snp-measure==0.0.7 \ - --hash=sha256:2625dab37898e9658b25b646d4a3bcceee1fb1b6b94d71bb7f59350faf3753ed \ - --hash=sha256:503ce35ea7469f1751233c69820c329d38c6d61a09cb3eedbfd591a1438464a4 - # via -r input.txt -types-cryptography==3.3.23.2 \ - --hash=sha256:09cc53f273dd4d8c29fa7ad11fefd9b734126d467960162397bc5e3e604dea75 \ - --hash=sha256:b965d548f148f8e87f353ccf2b7bd92719fdf6c845ff7cedf2abb393a0643e4f - # via sev-snp-measure diff --git a/.github/workflows/build-bazel-container.yml b/.github/workflows/build-bazel-container.yml deleted file mode 100644 index d680a8b14..000000000 --- a/.github/workflows/build-bazel-container.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build bazel dev container - -on: - push: - branches: - - "main" - paths: - - "bazel/container/**" - - ".github/workflows/build-bazel-container.yml" - workflow_dispatch: - -jobs: - build-container: - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - - name: Determine version - id: version - working-directory: ./bazel/container - run: | - version=$(grep "ARG BAZEL_VERSION" Containerfile | cut -d= -f2) - echo "version=v${version}" | tee -a "$GITHUB_OUTPUT" - - - name: Docker meta - id: meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 - with: - images: | - ghcr.io/edgelesssys/bazel-container - flavor: | - latest=false - tags: | - type=raw,value=${{ steps.version.outputs.version }},enable=${{ github.ref_name == 'main' }} - type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != 'main' }} - type=sha,value=${{ github.sha }} - type=raw,value=latest,enable=${{ github.ref_name == 'main' }} - - - name: Log in to the Container registry - uses: ./.github/actions/container_registry_login - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push container image - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 - with: - context: ./bazel/container - file: ./bazel/container/Containerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 46fd9cab8..a9ed3b89b 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -22,7 +22,7 @@ jobs: runs-on: [arc-runner-set] steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} @@ -31,7 +31,6 @@ jobs: with: useCache: "rbe" rbePlatform: "ubuntu-22.04" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Build all shell: bash diff --git a/.github/workflows/build-ccm-gcp.yml b/.github/workflows/build-ccm-gcp.yml index d28b23daf..b84514a1c 100644 --- a/.github/workflows/build-ccm-gcp.yml +++ b/.github/workflows/build-ccm-gcp.yml @@ -13,30 +13,30 @@ on: jobs: find-ccm-versions: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: versions: ${{ steps.find-versions.outputs.versions }} latest: ${{ steps.find-latest.outputs.latest }} steps: - name: Checkout Constellation - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Checkout kubernetes/cloud-provider-gcp - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "kubernetes/cloud-provider-gcp" path: "cloud-provider-gcp" fetch-depth: 0 - name: Setup Go environment - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: "1.21.5" + go-version: "1.24.3" cache: false - name: Install Crane run: | - go install github.com/google/go-containerregistry/cmd/crane@latest + go install github.com/google/go-containerregistry/cmd/crane@c195f151efe3369874c72662cd69ad43ee485128 # v0.20.2 - name: Find versions id: find-versions @@ -54,7 +54,7 @@ jobs: build-ccm-gcp: # matrix cannot handle empty lists if: needs.find-ccm-versions.outputs.versions != '[]' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: contents: read packages: write @@ -65,10 +65,10 @@ jobs: version: ${{ fromJson(needs.find-ccm-versions.outputs.versions) }} steps: - name: Checkout Constellation - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Checkout kubernetes/cloud-provider-gcp - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "kubernetes/cloud-provider-gcp" path: "cloud-provider-gcp" @@ -76,7 +76,7 @@ jobs: - name: Docker meta id: meta - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 with: images: | ghcr.io/edgelesssys/cloud-provider-gcp @@ -113,7 +113,7 @@ jobs: - name: Build and push container image id: build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: ./cloud-provider-gcp push: ${{ github.ref_name == 'main' }} diff --git a/.github/workflows/build-gcp-guest-agent.yml b/.github/workflows/build-gcp-guest-agent.yml index 6da99cf47..4fab1d2c4 100644 --- a/.github/workflows/build-gcp-guest-agent.yml +++ b/.github/workflows/build-gcp-guest-agent.yml @@ -10,7 +10,7 @@ env: jobs: build-gcp-guest-agent: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: contents: read packages: write @@ -69,7 +69,7 @@ jobs: - name: Checkout GoogleCloudPlatform/guest-agent if: steps.needs-build.outputs.out == 'true' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "GoogleCloudPlatform/guest-agent" ref: refs/tags/${{ steps.latest-release.outputs.latest }} @@ -77,7 +77,7 @@ jobs: - name: Checkout Constellation if: steps.needs-build.outputs.out == 'true' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: "constellation" ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} @@ -85,7 +85,7 @@ jobs: - name: Docker meta id: meta if: steps.needs-build.outputs.out == 'true' - uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 with: images: | ${{ env.REGISTRY }}/edgelesssys/gcp-guest-agent @@ -114,7 +114,7 @@ jobs: - name: Build and push container image if: steps.needs-build.outputs.out == 'true' id: build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: ./guest-agent file: ./constellation/3rdparty/gcp-guest-agent/Dockerfile diff --git a/.github/workflows/build-libvirt-container.yml b/.github/workflows/build-libvirt-container.yml index 91132fd78..625d6939d 100644 --- a/.github/workflows/build-libvirt-container.yml +++ b/.github/workflows/build-libvirt-container.yml @@ -13,18 +13,17 @@ on: jobs: build-container: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup bazel uses: ./.github/actions/setup_bazel_nix with: - useCache: "false" nixTools: | crane gzip diff --git a/.github/workflows/build-logcollector-images.yml b/.github/workflows/build-logcollector-images.yml index ddc5424ff..15517975f 100644 --- a/.github/workflows/build-logcollector-images.yml +++ b/.github/workflows/build-logcollector-images.yml @@ -13,14 +13,14 @@ on: jobs: build-logcollector-debugd-images: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: contents: read packages: write steps: - name: Check out repository id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} diff --git a/.github/workflows/build-os-image-scheduled.yml b/.github/workflows/build-os-image-scheduled.yml index f6c7332eb..e42c2ebfa 100644 --- a/.github/workflows/build-os-image-scheduled.yml +++ b/.github/workflows/build-os-image-scheduled.yml @@ -4,15 +4,15 @@ on: workflow_dispatch: schedule: - cron: "0 21 * * 2" # At 21:00 on Tuesday. - - cron: "10 21 * * 2" # At 21:10 on Tuesday. - cron: "20 21 * * 2" # At 21:20 on Tuesday. + - cron: "40 21 * * 2" # At 21:40 on Tuesday. - cron: "0 21 * * 4" # At 21:00 on Thursday. - - cron: "10 21 * * 4" # At 21:10 on Thursday. - cron: "20 21 * * 4" # At 21:20 on Thursday. + - cron: "40 21 * * 4" # At 21:40 on Thursday. jobs: stream: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: stream: ${{ steps.stream.outputs.stream }} steps: @@ -28,10 +28,10 @@ jobs: "0 21 * * 4" | "0 21 * * 2") echo "stream=debug" | tee -a "$GITHUB_OUTPUT" ;; - "10 21 * * 4" | "10 21 * * 2") + "20 21 * * 4" | "20 21 * * 2") echo "stream=console" | tee -a "$GITHUB_OUTPUT" ;; - "20 21 * * 4" | "20 21 * * 2") + "40 21 * * 4" | "40 21 * * 2") echo "stream=nightly" | tee -a "$GITHUB_OUTPUT" ;; *) @@ -54,22 +54,20 @@ jobs: update-code: # On nightly stream only. - if: | - github.event_name == 'workflow_dispatch' || - github.event.schedule == '20 21 * * 4' || - github.event.schedule == '20 21 * * 2' - needs: build-image - runs-on: ubuntu-22.04 + if: needs.stream.outputs.stream == 'nightly' + needs: ["build-image", "stream"] + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} + token: ${{ secrets.CI_COMMIT_PUSH_PR }} - name: Setup Go environment - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: "1.21.5" + go-version: "1.24.3" cache: false - name: Determine version @@ -99,7 +97,7 @@ jobs: run: rm -f internal/attestation/measurements/measurement-generator/generate - name: Create pull request - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: branch: "image/automated/update-measurements-${{ github.run_number }}" base: main @@ -111,6 +109,7 @@ jobs: It updates the hardcoded measurements and the image version (for QEMU/MiniConstellation). commit-message: "image: update measurements and image version" committer: edgelessci + author: edgelessci labels: no changelog # We need to push changes using a token, otherwise triggers like on:push and on:pull_request won't work. token: ${{ !github.event.pull_request.head.repo.fork && secrets.CI_COMMIT_PUSH_PR || '' }} @@ -118,10 +117,10 @@ jobs: notify-failure: if: failure() needs: [ "stream", "build-image", "update-code" ] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} diff --git a/.github/workflows/build-os-image.yml b/.github/workflows/build-os-image.yml index 38804419d..50783089a 100644 --- a/.github/workflows/build-os-image.yml +++ b/.github/workflows/build-os-image.yml @@ -47,7 +47,7 @@ on: jobs: build-settings: name: "Determine build settings" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: ref: ${{ steps.ref.outputs.ref }} stream: ${{ steps.stream.outputs.stream }} @@ -59,7 +59,7 @@ jobs: cliApiBasePath: ${{ steps.image-version.outputs.cliApiBasePath }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -129,110 +129,16 @@ jobs: echo "imageNameShort=ref/${REF}/stream/${STREAM}/${IMAGE_VERSION}" | tee -a "$GITHUB_OUTPUT" fi - make-os-image: - name: "Build OS using mkosi" + upload-os-image: + name: "Build OS using mkosi and upload it to CSPs" needs: [build-settings] runs-on: ubuntu-latest-8-cores - strategy: - fail-fast: false - matrix: - include: - - csp: aws - attestation_variant: aws-nitro-tpm - - csp: aws - attestation_variant: aws-sev-snp - - csp: azure - attestation_variant: azure-sev-snp - - csp: gcp - attestation_variant: gcp-sev-es - - csp: gcp - attestation_variant: gcp-sev-snp - - csp: qemu - attestation_variant: qemu-vtpm - - csp: openstack - attestation_variant: qemu-vtpm - steps: - - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - with: - ref: ${{ inputs.ref || github.head_ref }} - - - uses: ./.github/actions/setup_bazel_nix - with: - useCache: "false" - - - name: Build - id: build - shell: bash - working-directory: ${{ github.workspace }}/image - env: - TARGET: //image/system:${{ matrix.csp }}_${{ matrix.attestation_variant }}_${{ needs.build-settings.outputs.stream }} - run: | - echo "::group::Build" - bazel build //image/base:rpmdb - bazel build "${TARGET}" - { - echo "image-dir=$(bazel cquery --output=files "$TARGET")" - echo "rpmdb=$(bazel cquery --output=files //image/base:rpmdb)" - } | tee -a "$GITHUB_OUTPUT" - echo "::endgroup::" - - - name: Upload raw OS image as artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: image-${{ matrix.csp }}-${{ matrix.attestation_variant }} - path: ${{ steps.build.outputs.image-dir }}/constellation.raw - - - name: Upload individual OS parts as artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: parts-${{ matrix.csp }}-${{ matrix.attestation_variant }} - path: | - ${{ steps.build.outputs.image-dir }}/constellation.efi - ${{ steps.build.outputs.image-dir }}/constellation.initrd - ${{ steps.build.outputs.image-dir }}/constellation.vmlinuz - - - name: Upload sbom info as artifact - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - with: - name: sbom-${{ matrix.csp }}-${{ matrix.attestation_variant }} - path: ${{ steps.build.outputs.rpmdb }} - - upload-os-image: - name: "Upload OS image to CSP" - needs: [build-settings, make-os-image] - runs-on: ubuntu-22.04 permissions: id-token: write contents: read - strategy: - fail-fast: false - matrix: - include: - - csp: aws - attestation_variant: aws-nitro-tpm - - csp: aws - attestation_variant: aws-sev-snp - - csp: azure - attestation_variant: azure-sev-snp - - csp: gcp - attestation_variant: gcp-sev-es - - csp: gcp - attestation_variant: gcp-sev-snp - - csp: qemu - attestation_variant: qemu-vtpm - - csp: openstack - attestation_variant: qemu-vtpm - env: - RAW_IMAGE_PATH: mkosi.output.${{ matrix.csp }}_${{ matrix.attestation_variant }}/fedora~38/constellation.raw - JSON_OUTPUT: mkosi.output.${{ matrix.csp }}_${{ matrix.attestation_variant }}/fedora~38/image-upload.json - AZURE_IMAGE_PATH: mkosi.output.azure_${{ matrix.attestation_variant }}/fedora~38/image.vhd - GCP_IMAGE_PATH: mkosi.output.gcp_${{ matrix.attestation_variant }}/fedora~38/image.tar.gz - SHORTNAME: ${{ needs.build-settings.outputs.imageNameShort }} - ATTESTATION_VARIANT: ${{ matrix.attestation_variant }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -240,466 +146,85 @@ jobs: with: useCache: "false" - - name: Download OS image artifact - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: image-${{ matrix.csp }}-${{ matrix.attestation_variant }} - path: ${{ github.workspace }}/image/mkosi.output.${{ matrix.csp }}_${{ matrix.attestation_variant }}/fedora~38 - - - name: Install tools - shell: bash - run: | - echo "::group::Install tools" - sudo apt-get update - sudo apt-get install -y \ - pigz \ - qemu-utils \ - python3-pip - echo "::endgroup::" - - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline aws-region: eu-central-1 - name: Login to Azure - if: matrix.csp == 'azure' uses: ./.github/actions/login_azure with: azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} - name: Login to GCP - if: matrix.csp == 'gcp' uses: ./.github/actions/login_gcp with: service_account: "image-uploader@constellation-images.iam.gserviceaccount.com" - - name: Upload AWS image - if: matrix.csp == 'aws' + - name: Login to OpenStack + uses: ./.github/actions/login_openstack + with: + clouds_yaml: ${{ secrets.STACKIT_IMAGE_UPLOAD_CLOUDS_YAML }} + + - name: Allow unrestricted user namespaces + shell: bash + run: | + sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_unconfined=0 + sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Build and upload + id: build shell: bash working-directory: ${{ github.workspace }}/image - run: | - echo "::group::Upload AWS image" - bazel run //image/upload -- image aws \ - --verbose \ - --raw-image "${RAW_IMAGE_PATH}" \ - --attestation-variant "${ATTESTATION_VARIANT}" \ - --version "${SHORTNAME}" \ - --out "${JSON_OUTPUT}" - echo -e "Uploaded AWS image: \n\n\`\`\`\n$(jq < "${JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" - echo "::endgroup::" - - - name: Upload GCP image - if: matrix.csp == 'gcp' - shell: bash - working-directory: ${{ github.workspace }}/image - run: | - echo "::group::Upload GCP image" - upload/pack.sh gcp "${RAW_IMAGE_PATH}" "${GCP_IMAGE_PATH}" - bazel run //image/upload -- image gcp \ - --verbose \ - --raw-image "${GCP_IMAGE_PATH}" \ - --attestation-variant "${ATTESTATION_VARIANT}" \ - --version "${SHORTNAME}" \ - --out "${JSON_OUTPUT}" - echo -e "Uploaded GCP image: \n\n\`\`\`\n$(jq < "${JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" - echo "::endgroup::" - - - name: Upload Azure image - if: matrix.csp == 'azure' - shell: bash - working-directory: ${{ github.workspace }}/image - run: | - echo "::group::Upload Azure image" - upload/pack.sh azure "${RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}" - bazel run //image/upload -- image azure \ - --verbose \ - --raw-image "${AZURE_IMAGE_PATH}" \ - --attestation-variant "${ATTESTATION_VARIANT}" \ - --version "${SHORTNAME}" \ - --out "${JSON_OUTPUT}" - echo -e "Uploaded Azure image: \n\n\`\`\`\n$(jq < "${JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" - echo "::endgroup::" - - - name: Upload OpenStack image - if: matrix.csp == 'openstack' - shell: bash - working-directory: ${{ github.workspace }}/image - run: | - echo "::group::Upload OpenStack image" - bazel run //image/upload -- image openstack \ - --verbose \ - --raw-image "${RAW_IMAGE_PATH}" \ - --attestation-variant "${ATTESTATION_VARIANT}" \ - --version "${SHORTNAME}" \ - --out "${JSON_OUTPUT}" - echo -e "Uploaded OpenStack image: \n\n\`\`\`\n$(jq < "${JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" - echo "::endgroup::" - - - name: Upload QEMU image - if: matrix.csp == 'qemu' - shell: bash - working-directory: ${{ github.workspace }}/image - run: | - echo "::group::Upload QEMU image" - bazel run //image/upload -- image qemu \ - --verbose \ - --raw-image "${RAW_IMAGE_PATH}" \ - --attestation-variant "${ATTESTATION_VARIANT}" \ - --version "${SHORTNAME}" \ - --out "${JSON_OUTPUT}" - echo -e "Uploaded QEMU image: \n\n\`\`\`\n$(jq < "${JSON_OUTPUT}")\n\`\`\`\n" >> "$GITHUB_STEP_SUMMARY" - echo "::endgroup::" - - - name: Upload image lookup table as artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: lookup-table - path: ${{ github.workspace }}/image/mkosi.output.*/*/image-upload*.json - - calculate-pcrs: - name: "Calculate PCRs" - needs: [build-settings, make-os-image] - permissions: - id-token: write - contents: read - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - include: - - csp: aws - attestation_variant: aws-nitro-tpm - - csp: aws - attestation_variant: aws-sev-snp - - csp: azure - attestation_variant: azure-sev-snp - - csp: gcp - attestation_variant: gcp-sev-es - - csp: gcp - attestation_variant: gcp-sev-snp - - csp: qemu - attestation_variant: qemu-vtpm - - csp: openstack - attestation_variant: qemu-vtpm - steps: - - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - with: - ref: ${{ inputs.ref || github.head_ref }} - - - name: Download OS image artifact - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: image-${{ matrix.csp }}-${{ matrix.attestation_variant }} - - - uses: ./.github/actions/setup_bazel_nix - with: - useCache: "false" - - - name: Install dependencies - run: | - echo "::group::Install dependencies" - sudo apt-get update - sudo apt-get install -y systemd-container # for systemd-dissect - echo "::endgroup::" - - - name: Calculate expected PCRs - working-directory: ${{ github.workspace }}/image/measured-boot - run: | - echo "::group::Calculate expected PCRs" - bazel run --run_under="sudo -E" //image/measured-boot/cmd ${{ github.workspace }}/constellation.raw ${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json >> "$GITHUB_STEP_SUMMARY" - echo "::endgroup::" - - - name: Add static PCRs - run: | - case ${{ matrix.csp }} in - aws) - yq e '.csp = "AWS" | - .attestationVariant = "${{ matrix.attestation_variant }}" | - .measurements.0.warnOnly = true | - .measurements.0.expected = "737f767a12f54e70eecbc8684011323ae2fe2dd9f90785577969d7a2013e8c12" | - .measurements.2.warnOnly = true | - .measurements.2.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.3.warnOnly = true | - .measurements.3.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.4.warnOnly = false | - .measurements.6.warnOnly = true | - .measurements.6.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.8.warnOnly = false | - .measurements.9.warnOnly = false | - .measurements.11.warnOnly = false | - .measurements.12.warnOnly = false | - .measurements.13.warnOnly = false | - .measurements.14.warnOnly = true | - .measurements.14.expected = "0000000000000000000000000000000000000000000000000000000000000000" | - .measurements.15.warnOnly = false' \ - -I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" - ;; - azure) - yq e '.csp = "Azure" | - .attestationVariant = "${{ matrix.attestation_variant }}" | - .measurements.1.warnOnly = true | - .measurements.1.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.2.warnOnly = true | - .measurements.2.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.3.warnOnly = true | - .measurements.3.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.4.warnOnly = false | - .measurements.8.warnOnly = false | - .measurements.9.warnOnly = false | - .measurements.11.warnOnly = false | - .measurements.12.warnOnly = false | - .measurements.13.warnOnly = false | - .measurements.14.warnOnly = true | - .measurements.14.expected = "0000000000000000000000000000000000000000000000000000000000000000" | - .measurements.15.warnOnly = false' \ - -I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" - ;; - gcp) - yq e '.csp = "GCP" | - .attestationVariant = "${{ matrix.attestation_variant }}" | - .measurements.1.warnOnly = true | - .measurements.1.expected = "745f2fb4235e4647aa0ad5ace781cd929eb68c28870e7dd5d1a1535854325e56" | - .measurements.2.warnOnly = true | - .measurements.2.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.3.warnOnly = true | - .measurements.3.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.4.warnOnly = false | - .measurements.6.warnOnly = true | - .measurements.6.expected = "3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969" | - .measurements.8.warnOnly = false | - .measurements.9.warnOnly = false | - .measurements.11.warnOnly = false | - .measurements.12.warnOnly = false | - .measurements.13.warnOnly = false | - .measurements.14.warnOnly = true | - .measurements.14.expected = "0000000000000000000000000000000000000000000000000000000000000000" | - .measurements.15.warnOnly = false' \ - -I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" - ;; - openstack) - yq e '.csp = "OpenStack" | - .attestationVariant = "${{ matrix.attestation_variant }}" | - .measurements.4.warnOnly = false | - .measurements.8.warnOnly = false | - .measurements.9.warnOnly = false | - .measurements.11.warnOnly = false | - .measurements.12.warnOnly = false | - .measurements.13.warnOnly = false | - .measurements.14.warnOnly = true | - .measurements.14.expected = "0000000000000000000000000000000000000000000000000000000000000000" | - .measurements.15.warnOnly = false' \ - -I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" - ;; - qemu) - yq e '.csp = "QEMU" | - .attestationVariant = "${{ matrix.attestation_variant }}" | - .measurements.4.warnOnly = false | - .measurements.8.warnOnly = false | - .measurements.9.warnOnly = false | - .measurements.11.warnOnly = false | - .measurements.12.warnOnly = false | - .measurements.13.warnOnly = false | - .measurements.14.warnOnly = true | - .measurements.14.expected = "0000000000000000000000000000000000000000000000000000000000000000" | - .measurements.15.warnOnly = false' \ - -I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" - ;; - *) - echo "Unknown CSP: ${{ matrix.csp }}" - exit 1 - ;; - esac - - # TODO (malt3): Calculate PCR from firmware blob. - # AWS SNP machines have a different expected value for PCR 0. - if [[ ${{ matrix.attestation_variant }} = "aws-sev-snp" ]] - then - yq e '.csp = "AWS" | - .measurements.0.expected = "7b068c0c3ac29afe264134536b9be26f1d4ccd575b88d3c3ceabf36ac99c0278"' \ - -I 0 -o json -i "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" - fi - - - name: Envelope measurements - shell: bash - run: | - echo "::group::Envelope measurements" - bazel run //image/upload -- measurements envelope \ - --in "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" \ - --out "${{ github.workspace }}/pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json" \ - --version "${{ needs.build-settings.outputs.imageNameShort }}" \ - --csp "${{ matrix.csp }}" \ - --attestation-variant "${{ matrix.attestation_variant }}" - echo "::endgroup::" - - - name: Upload expected measurements as artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: measurements - path: pcrs-${{ matrix.csp }}-${{ matrix.attestation_variant }}.json - - upload-pcrs: - name: "Sign & upload PCRs" - needs: [build-settings, calculate-pcrs] - permissions: - id-token: write - contents: read - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - with: - ref: ${{ inputs.ref || github.head_ref }} - - - uses: ./.github/actions/setup_bazel_nix - with: - useCache: "false" - - - name: Download measurements - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: measurements - - - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline - aws-region: eu-central-1 - - - name: Install Cosign - uses: sigstore/cosign-installer@c85d0e205a72a294fe064f618a87dbac13084086 # v2.8.1 - - - name: Install Rekor - shell: bash - run: | - curl -fsSLO https://github.com/sigstore/rekor/releases/download/v0.12.0/rekor-cli-linux-amd64 - sudo install rekor-cli-linux-amd64 /usr/local/bin/rekor-cli - rm rekor-cli-linux-amd64 - - - name: Merge measurements - shell: bash - run: | - echo "::group::Merge measurements" - bazel run //image/upload -- measurements merge \ - --out measurements.json \ - pcrs-*.json - echo "::endgroup::" - - - name: Sign measurements - if: inputs.stream != 'debug' - shell: bash env: + TARGET: //image/system:upload_${{ needs.build-settings.outputs.stream }} + REF: ${{ needs.build-settings.outputs.ref }} + STREAM: ${{ needs.build-settings.outputs.stream }} + SHORT_NAME: ${{ needs.build-settings.outputs.imageNameShort }} COSIGN_PUBLIC_KEY: ${{ inputs.isRelease && secrets.COSIGN_PUBLIC_KEY || secrets.COSIGN_DEV_PUBLIC_KEY }} COSIGN_PRIVATE_KEY: ${{ inputs.isRelease && secrets.COSIGN_PRIVATE_KEY || secrets.COSIGN_DEV_PRIVATE_KEY }} COSIGN_PASSWORD: ${{ inputs.isRelease && secrets.COSIGN_PASSWORD || secrets.COSIGN_DEV_PASSWORD }} run: | + echo "::group::Build" echo "${COSIGN_PUBLIC_KEY}" > cosign.pub - # Enabling experimental mode also publishes signature to Rekor - COSIGN_EXPERIMENTAL=1 cosign sign-blob --key env://COSIGN_PRIVATE_KEY \ - "${{ github.workspace }}/measurements.json" > "${{ github.workspace }}/measurements.json.sig" - # Verify - As documentation & check - # Local Signature (input: artifact, key, signature) - cosign verify-blob --key cosign.pub \ - --signature "measurements.json.sig" \ - "measurements.json" - # Transparency Log Signature (input: artifact, key) - uuid=$(rekor-cli search --artifact "${{ github.workspace }}/measurements.json" | tail -n 1) - sig=$(rekor-cli get --uuid="${uuid}" --format=json | jq -r .Body.HashedRekordObj.signature.content) - cosign verify-blob --key cosign.pub --signature <(echo "${sig}") "${{ github.workspace }}/measurements.json" - - - name: Create stub signature file - if: inputs.stream == 'debug' - shell: bash - run: | - echo "THOSE MEASUREMENTS BELONG TO A DEBUG IMAGE. THOSE ARE NOT SINGED BY ANY KEY." > "${{ github.workspace }}/measurements.json.sig" - - - name: Upload measurements - shell: bash - run: | - echo "::group::Upload measurements" - bazel run //image/upload -- measurements upload \ - --measurements measurements.json \ - --signature measurements.json.sig + COSIGN_PUBLIC_KEY_PATH="$(realpath ./cosign.pub)" + export COSIGN_PUBLIC_KEY_PATH + opts=( + --ref "${REF}" + --upload-measurements + ) + if [[ "${STREAM}" = "debug" ]]; then + opts+=(--fake-sign) + fi + bazel build //image/base:rpmdb + bazel run "${TARGET}" -- "${opts[@]}" + { + echo "rpmdb=$(bazel cquery --output=files //image/base:rpmdb)" + } | tee -a "$GITHUB_OUTPUT" + echo -ne "Uploaded OS image:\n\n\`\`\`\n${SHORT_NAME}\n\`\`\`" | tee -a "$GITHUB_STEP_SUMMARY" echo "::endgroup::" - upload-sbom: - name: "Upload SBOM" - needs: [build-settings, make-os-image] - permissions: - id-token: write - contents: read - runs-on: ubuntu-22.04 - steps: - - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline - aws-region: eu-central-1 - - - name: Download sbom - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - # downloading / using only the QEMU manifest is fine - # since the images only differ in the ESP partition - name: sbom-qemu-qemu-vtpm - - - name: Upload SBOMs to S3 + - name: Upload SBOM to S3 shell: bash + env: + RPMDB: ${{ steps.build.outputs.rpmdb }} run: | aws s3 cp \ - rpmdb.tar \ + "${RPMDB}" \ "s3://cdn-constellation-backend/${{needs.build-settings.outputs.imageApiBasePath}}/${file}" \ --no-progress - upload-artifacts: - name: "Upload image lookup table and CLI compatibility info" - runs-on: ubuntu-22.04 - needs: [build-settings, upload-os-image] - permissions: - id-token: write - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - with: - ref: ${{ inputs.ref || github.head_ref }} - - - uses: ./.github/actions/setup_bazel_nix - with: - useCache: "false" - - - name: Download image lookup table - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3 - with: - name: lookup-table - - - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline - aws-region: eu-central-1 - - - name: Upload lookup table to S3 - shell: bash - run: bazel run //image/upload -- info --verbose mkosi.output.*/*/image-upload*.json - - - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - with: - ref: ${{ inputs.ref || github.head_ref }} - - name: Create CLI compatibility information artifact shell: bash run: | bazel run //hack/cli-k8s-compatibility -- \ --ref=${{ needs.build-settings.outputs.ref }} \ --stream=${{ needs.build-settings.outputs.stream }} \ - --version=${{ needs.build-settings.outputs.imageVersion }} \ + --version=${{ needs.build-settings.outputs.imageVersion }} add-image-version-to-versionsapi: - needs: [upload-artifacts, upload-pcrs, build-settings] + needs: [upload-os-image, build-settings] name: "Add image version to versionsapi" if: needs.build-settings.outputs.ref != '-' permissions: @@ -715,7 +240,7 @@ jobs: add_latest: true add-cli-version-to-versionsapi: - needs: [upload-artifacts, build-settings, add-image-version-to-versionsapi] + needs: [upload-os-image, build-settings, add-image-version-to-versionsapi] name: "Add CLI version to versionsapi" if: needs.build-settings.outputs.ref != '-' permissions: diff --git a/.github/workflows/build-versionsapi-ci-image.yml b/.github/workflows/build-versionsapi-ci-image.yml deleted file mode 100644 index 0ccd1f987..000000000 --- a/.github/workflows/build-versionsapi-ci-image.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build and upload versionsapi CI image - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - "internal/api/versionsapi/**" - - ".github/workflows/build-versionsapi-ci-image.yml" - - ".github/actions/versionsapi/**" - -jobs: - build-versionsapi-ci-cli: - runs-on: ubuntu-22.04 - permissions: - contents: read - packages: write - steps: - - name: Check out repository - id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - with: - ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - - - name: Build and upload container image - uses: ./.github/actions/build_micro_service - with: - name: versionsapi-ci-cli - dockerfile: .github/actions/versionsapi/Dockerfile - githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index ca663ee00..598a64445 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -17,17 +17,15 @@ on: jobs: linkChecker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Link Checker - uses: lycheeverse/lychee-action@ec3ed119d4f44ad2673a7232460dc7dff59d2421 # v1.8.0 + uses: lycheeverse/lychee-action@82202e5e9c2f4ef1a55a3d02563e1cb6041e5332 # v2.4.1 with: - args: "--verbose --no-progress --max-concurrency 5 --exclude-path './internal/constellation/helm/charts/cilium' './**/*.md' './**/*.html'" + args: "--config ./.lychee.toml './**/*.md' './**/*.html'" fail: true - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/check-measurements-reproducibility.yml b/.github/workflows/check-measurements-reproducibility.yml new file mode 100644 index 000000000..3a91eda81 --- /dev/null +++ b/.github/workflows/check-measurements-reproducibility.yml @@ -0,0 +1,27 @@ +name: Check measurements reproducibility +on: + workflow_dispatch: + inputs: + version: + type: string + description: The version of the measurements that are downloaded from the CDN. + required: true + ref: + type: string + description: The git ref to check out. You probably want this to be the tag of the release you are testing. + required: true + +jobs: + check-reproducibility: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Check reproducibility + uses: ./.github/actions/check_measurements_reproducibility + with: + version: ${{ github.event.inputs.version }} + ref: ${{ github.event.inputs.ref }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 551d97a3e..8c77ddacb 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -17,7 +17,7 @@ on: jobs: codeql: name: CodeQL - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: # Force CodeQL to run the extraction on the files compiled by our custom # build command, as opposed to letting the autobuilder figure it out. @@ -34,17 +34,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Go environment if: matrix.language == 'go' - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: "1.21.5" + go-version: "1.24.3" cache: false - name: Initialize CodeQL - uses: github/codeql-action/init@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1 + uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 with: languages: ${{ matrix.language }} @@ -63,6 +63,6 @@ jobs: echo "::endgroup::" - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1 + uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/docs-vale.yml b/.github/workflows/docs-vale.yml index 5e2e03acd..bb6331ce5 100644 --- a/.github/workflows/docs-vale.yml +++ b/.github/workflows/docs-vale.yml @@ -12,20 +12,21 @@ on: - "docs/**" jobs: - prose: - runs-on: ubuntu-22.04 + vale: + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - + # Work around https://github.com/errata-ai/vale-action/issues/128. + - run: | + venv="$HOME/.local/share/venv" + python3 -m venv "$venv" + echo "$venv/bin" >> "$GITHUB_PATH" - name: Vale - uses: errata-ai/vale-action@c4213d4de3d5f718b8497bd86161531c78992084 # tag=v2.0.1 + uses: errata-ai/vale-action@2690bc95f0ed3cb5220492575af09c51b04fbea9 # tag=reviewdog with: - version: 2.17.0 files: docs/docs - env: - # Required, set by GitHub actions automatically: - # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + fail_on_error: true + version: 3.9.3 diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml index c2bb03acb..84b696afb 100644 --- a/.github/workflows/draft-release.yml +++ b/.github/workflows/draft-release.yml @@ -50,7 +50,7 @@ on: jobs: build-cli: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -72,7 +72,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -92,8 +92,8 @@ jobs: cosignPassword: ${{ inputs.key == 'release' && secrets.COSIGN_PASSWORD || secrets.COSIGN_DEV_PASSWORD }} - name: Upload CLI as artifact (unix) - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - if : ${{ matrix.os != 'windows' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ matrix.os != 'windows' }} with: name: constellation-${{ matrix.os }}-${{ matrix.arch }} path: | @@ -101,8 +101,8 @@ jobs: build/constellation-${{ matrix.os }}-${{ matrix.arch }}.sig - name: Upload CLI as artifact (windows) - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - if : ${{ matrix.os == 'windows' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ matrix.os == 'windows' }} with: name: constellation-${{ matrix.os }}-${{ matrix.arch }} path: | @@ -110,7 +110,7 @@ jobs: build/constellation-${{ matrix.os }}-${{ matrix.arch }}.exe.sig build-terraform-provider: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -133,7 +133,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -149,27 +149,27 @@ jobs: targetArch: ${{ matrix.arch }} - name: Upload Terraform Provider Binary as artifact (unix) - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - if : ${{ matrix.os != 'windows' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ matrix.os != 'windows' }} with: name: terraform-provider-constellation-${{ matrix.os }}-${{ matrix.arch }} path: | build/terraform-provider-constellation-${{ matrix.os }}-${{ matrix.arch }} - name: Upload Terraform Provider Binary as artifact (windows) - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - if : ${{ matrix.os == 'windows' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ matrix.os == 'windows' }} with: name: terraform-provider-constellation-${{ matrix.os }}-${{ matrix.arch }} path: | build/terraform-provider-constellation-${{ matrix.os }}-${{ matrix.arch }}.exe upload-terraform-module: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -177,7 +177,7 @@ jobs: uses: ./.github/actions/upload_terraform_module push-containers: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: inputs.pushContainers permissions: actions: read @@ -187,7 +187,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -208,7 +208,7 @@ jobs: run: bazel run //bazel/release:push provenance-subjects: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - build-cli - signed-sbom @@ -219,7 +219,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -227,7 +227,7 @@ jobs: uses: ./.github/actions/download_release_binaries - name: Download CLI SBOM - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation.spdx.sbom @@ -252,16 +252,16 @@ jobs: echo provenance-subjects="${HASHESB64}" >> "$GITHUB_OUTPUT" signed-sbom: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} - name: Install Cosign - uses: sigstore/cosign-installer@c85d0e205a72a294fe064f618a87dbac13084086 # v2.8.1 + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 - name: Download Syft & Grype uses: ./.github/actions/install_syft_grype @@ -287,7 +287,7 @@ jobs: - name: Build signed SBOM run: | syft build/constellation-linux-amd64 --catalogers go-module --file constellation.spdx.sbom -o spdx-json - cosign sign-blob --key env://COSIGN_PRIVATE_KEY constellation.spdx.sbom > constellation.spdx.sbom.sig + cosign sign-blob --yes --key env://COSIGN_PRIVATE_KEY constellation.spdx.sbom > constellation.spdx.sbom.sig grype constellation.spdx.sbom --fail-on high --only-fixed --add-cpes-if-none env: COSIGN_EXPERIMENTAL: 1 @@ -296,13 +296,13 @@ jobs: COSIGN_PASSWORD: ${{ inputs.key == 'release' && secrets.COSIGN_PASSWORD || secrets.COSIGN_DEV_PASSWORD }} - name: Upload Constellation CLI SBOM - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: constellation.spdx.sbom path: constellation.spdx.sbom - name: Upload Constellation CLI SBOM's signature - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: constellation.spdx.sbom.sig path: constellation.spdx.sbom.sig @@ -316,14 +316,14 @@ jobs: - provenance-subjects # This must not be pinned to digest. See: # https://github.com/slsa-framework/slsa-github-generator#referencing-slsa-builders-and-generators - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.9.0 + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.1.0 with: base64-subjects: "${{ needs.provenance-subjects.outputs.provenance-subjects }}" provenance-verify: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: - SLSA_VERIFIER_VERSION: "2.0.1" + SLSA_VERIFIER_VERSION: "2.7.0" needs: - build-cli - provenance @@ -332,7 +332,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -340,12 +340,12 @@ jobs: uses: ./.github/actions/download_release_binaries - name: Download CLI SBOM - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation.spdx.sbom - name: Download provenance - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.provenance.outputs.provenance-name }} @@ -395,7 +395,7 @@ jobs: release: permissions: contents: write - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - build-cli - provenance @@ -405,7 +405,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.head_ref }} @@ -418,17 +418,17 @@ jobs: uses: ./.github/actions/download_release_binaries - name: Download CLI SBOM - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation.spdx.sbom - name: Download Constellation CLI SBOM's signature - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: constellation.spdx.sbom.sig - name: Download Constellation provenance - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.provenance.outputs.provenance-name }} @@ -454,9 +454,10 @@ jobs: if [[ "${ext}" = "exe" ]]; then cp "${file}" "${folder_name}/terraform-provider-constellation_v${version}.exe" else + chmod 755 "${file}" # the upload artifact does not preserve file permissions (https://github.com/actions/upload-artifact/tree/main/?tab=readme-ov-file#permission-loss) cp "${file}" "${folder_name}/terraform-provider-constellation_v${version}" fi - zip -r "${folder_name}.zip" "${folder_name}" + (cd "${folder_name}" && zip "../${folder_name}.zip" ./*) # do not zip the folder itself rm -r "${folder_name}" done @@ -471,7 +472,7 @@ jobs: - name: Create release with artifacts id: create-release # GitHub endorsed release project. See: https://github.com/actions/create-release - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 + uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 with: draft: true generate_release_notes: true @@ -486,7 +487,7 @@ jobs: terraform-module.zip - name: Create Terraform provider release with artifcats - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15 + uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2 with: draft: true generate_release_notes: false diff --git a/.github/workflows/e2e-attestationconfigapi.yml b/.github/workflows/e2e-attestationconfigapi.yml index 7fcca9028..3c3d233c1 100644 --- a/.github/workflows/e2e-attestationconfigapi.yml +++ b/.github/workflows/e2e-attestationconfigapi.yml @@ -9,10 +9,7 @@ on: paths: - "internal/api/**" - ".github/workflows/e2e-attestationconfigapi.yml" - pull_request: - paths: - - "internal/api/**" - - ".github/workflows/e2e-attestationconfigapi.yml" + - "go.mod" jobs: e2e-api: @@ -20,8 +17,8 @@ jobs: fail-fast: false max-parallel: 1 matrix: - csp: ["azure", "aws"] - runs-on: ubuntu-22.04 + attestationVariant: ["azure-sev-snp", "azure-tdx", "aws-sev-snp", "gcp-sev-snp"] + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -29,7 +26,7 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # Don't trigger in forks, use head on pull requests, use default otherwise. ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || github.event.pull_request.head.sha || '' }} @@ -37,7 +34,6 @@ jobs: - name: Run Attestationconfig API E2E uses: ./.github/actions/e2e_attestationconfigapi with: - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} cosignPrivateKey: ${{ secrets.COSIGN_DEV_PRIVATE_KEY }} cosignPassword: ${{ secrets.COSIGN_DEV_PASSWORD }} - csp: ${{ matrix.csp }} + attestationVariant: ${{ matrix.attestationVariant }} diff --git a/.github/workflows/e2e-cleanup.yml b/.github/workflows/e2e-cleanup.yml new file mode 100644 index 000000000..67e5bf0ff --- /dev/null +++ b/.github/workflows/e2e-cleanup.yml @@ -0,0 +1,26 @@ +name: e2e cleanup + +on: + schedule: + - cron: "0 0 * * *" # At 00:00 every day + workflow_dispatch: + + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + actions: read + id-token: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Cleanup + uses: ./.github/actions/e2e_cleanup_timeframe + with: + ghToken: ${{ secrets.GITHUB_TOKEN }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + azure_credentials: ${{ secrets.AZURE_E2E_DESTROY_CREDENTIALS }} + openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }} + stackitUat: ${{ secrets.STACKIT_CI_UAT }} diff --git a/.github/workflows/e2e-mini.yml b/.github/workflows/e2e-mini.yml index 74d6a715a..bf0cf1cad 100644 --- a/.github/workflows/e2e-mini.yml +++ b/.github/workflows/e2e-mini.yml @@ -20,7 +20,7 @@ on: jobs: e2e-mini: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 environment: e2e permissions: id-token: write @@ -29,12 +29,12 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.ref || github.event.workflow_run.head_branch || github.head_ref }} - name: Azure login OIDC - uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.7 + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 with: client-id: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -46,6 +46,6 @@ jobs: azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureTenantID: ${{ secrets.AZURE_TENANT_ID }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureIAMCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io githubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index 45efd049d..f7ea0ad11 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -12,7 +12,7 @@ jobs: matrix: refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] name: Find latest image - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -21,7 +21,7 @@ jobs: image-release-stable: ${{ steps.relabel-output.outputs.image-release-stable }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} @@ -45,31 +45,42 @@ jobs: fail-fast: false max-parallel: 5 matrix: - kubernetesVersion: ["1.27"] # should be default - provider: ["gcp", "azure", "aws"] + kubernetesVersion: ["v1.31"] # This should correspond to the current default k8s minor. + attestationVariant: ["gcp-sev-es", "gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] - test: ["sonobuoy full"] - runs-on: ubuntu-22.04 + test: ["sonobuoy quick"] + runs-on: ubuntu-24.04 permissions: id-token: write checks: write contents: read packages: write + actions: write needs: [find-latest-image] steps: - name: Check out repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Split attestationVariant + id: split-attestationVariant + shell: bash + run: | + attestationVariant="${{ matrix.attestationVariant }}" + cloudProvider="${attestationVariant%%-*}" + + echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" + - name: Run E2E test id: e2e_test uses: ./.github/actions/e2e_test with: workerNodesCount: "2" controlNodesCount: "3" - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} osImage: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }} isDebugImage: ${{ matrix.refStream == 'ref/main/stream/debug/?' }} cliVersion: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || '' }} @@ -79,7 +90,7 @@ jobs: gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" kubernetesVersion: ${{ matrix.kubernetesVersion }} test: ${{ matrix.test }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io @@ -91,6 +102,7 @@ jobs: awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} clusterCreation: "cli" + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() @@ -98,7 +110,7 @@ jobs: with: kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} clusterCreation: "cli" - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" @@ -106,10 +118,20 @@ jobs: if: always() uses: ./.github/actions/constellation_iam_destroy with: - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Notify about failure if: | failure() && @@ -122,12 +144,13 @@ jobs: refStream: ${{ matrix.refStream }} test: ${{ matrix.test }} kubernetesVersion: ${{ matrix.kubernetesVersion }} - provider: ${{ matrix.provider }} + provider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} clusterCreation: "cli" e2e-mini: name: Run miniconstellation E2E test - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 environment: e2e permissions: id-token: write @@ -136,12 +159,12 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Azure login OIDC - uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.7 + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 with: client-id: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -153,7 +176,7 @@ jobs: azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureTenantID: ${{ secrets.AZURE_TENANT_ID }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureIAMCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io githubToken: ${{ secrets.GITHUB_TOKEN }} @@ -166,5 +189,6 @@ jobs: uses: ./.github/actions/notify_e2e_failure with: projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} + attestationVariant: "qemu-vtpm" test: "MiniConstellation" provider: "QEMU" diff --git a/.github/workflows/e2e-test-internal-lb.yml b/.github/workflows/e2e-test-internal-lb.yml index 6fb1ca33e..ab36cec4a 100644 --- a/.github/workflows/e2e-test-internal-lb.yml +++ b/.github/workflows/e2e-test-internal-lb.yml @@ -7,22 +7,24 @@ on: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: choice options: - - "gcp" - - "azure" - - "aws" - default: "azure" + - "aws-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "gcp-sev-es" + - "gcp-sev-snp" + default: "azure-sev-snp" required: true runner: description: "Architecture of the runner that executes the CLI" type: choice options: - - "ubuntu-22.04" - - "macos-12" - default: "ubuntu-22.04" + - "ubuntu-24.04" + - "macos-latest" + default: "ubuntu-24.04" test: description: "The test to run." type: choice @@ -39,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.27" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." @@ -76,7 +77,7 @@ jobs: uses: ./.github/workflows/e2e-test.yml with: nodeCount: ${{ inputs.nodeCount }} - cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} runner: ${{ inputs.runner }} test: ${{ inputs.test }} kubernetesVersion: ${{ inputs.kubernetesVersion }} diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml index e89bb4e76..28e8e9310 100644 --- a/.github/workflows/e2e-test-marketplace-image.yml +++ b/.github/workflows/e2e-test-marketplace-image.yml @@ -7,20 +7,24 @@ on: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: choice options: - - "azure" - default: "azure" + - "aws-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "gcp-sev-es" + - "gcp-sev-snp" + default: "azure-sev-snp" required: true runner: description: "Architecture of the runner that executes the CLI" type: choice options: - - "ubuntu-22.04" - - "macos-12" - default: "ubuntu-22.04" + - "ubuntu-24.04" + - "macos-latest" + default: "ubuntu-24.04" test: description: "The test to run." type: choice @@ -37,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.27" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." @@ -74,7 +77,7 @@ jobs: uses: ./.github/workflows/e2e-test.yml with: nodeCount: ${{ inputs.nodeCount }} - cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} runner: ${{ inputs.runner }} test: ${{ inputs.test }} kubernetesVersion: ${{ inputs.kubernetesVersion }} diff --git a/.github/workflows/e2e-test-provider-example.yml b/.github/workflows/e2e-test-provider-example.yml new file mode 100644 index 000000000..43eacd005 --- /dev/null +++ b/.github/workflows/e2e-test-provider-example.yml @@ -0,0 +1,494 @@ +name: e2e test Terraform provider example + +on: + workflow_dispatch: + inputs: + ref: + type: string + description: "Git ref to checkout" + regionZone: + description: "Region or zone to create the cluster in. Leave empty for default region/zone." + type: string + image: + description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used." + type: string + providerVersion: + description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." + type: string + toImage: + description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. + type: string + required: false + toKubernetes: + description: Kubernetes version to target for the upgrade, empty for no upgrade. + type: string + required: false + attestationVariant: + description: "Attestation variant to use." + type: choice + options: + - "aws-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "gcp-sev-es" + - "gcp-sev-snp" + default: "azure-sev-snp" + required: true + workflow_call: + inputs: + ref: + type: string + description: "Git ref to checkout" + regionZone: + description: "Which zone to use." + type: string + image: + description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used." + type: string + providerVersion: + description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." + type: string + toImage: + description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. + type: string + required: false + toKubernetes: + description: Kubernetes version to target for the upgrade, empty for target's default version. + type: string + required: false + attestationVariant: + description: "Attestation variant to use." + type: string + required: true + +jobs: + provider-example-test: + runs-on: ubuntu-24.04 + permissions: + id-token: write + contents: read + packages: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.ref || github.head_ref }} + + - name: Get Latest Image + id: find-latest-image + uses: ./.github/actions/find_latest_image + with: + git-ref: ${{ inputs.ref }} + imageVersion: ${{ inputs.image }} + ref: main + stream: nightly + + - name: Determine cloudprovider from attestation variant + id: determine + shell: bash + run: | + attestationVariant="${{ inputs.attestationVariant }}" + cloudProvider="${attestationVariant%%-*}" + + echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" + + - name: Log in to the Container registry + uses: ./.github/actions/container_registry_login + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Download CLI # needed to determine K8s version for release versions + if: inputs.providerVersion != '' + shell: bash + run: | + curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/constellation-linux-amd64 + chmod u+x constellation + ./constellation version + mkdir -p ${{ github.workspace }}/release + cp ./constellation ${{ github.workspace }}/release + + - name: Setup bazel + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: terraform + + - name: Create prefix + id: create-prefix + shell: bash + run: | + uuid=$(uuidgen | tr "[:upper:]" "[:lower:]") + uuid="${uuid%%-*}" + uuid="${uuid: -3}" # Final resource name must be no longer than 10 characters on AWS + echo "uuid=${uuid}" | tee -a "${GITHUB_OUTPUT}" + echo "prefix=e2e-${uuid}" | tee -a "${GITHUB_OUTPUT}" + + - name: Build Constellation provider and CLI # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade + working-directory: ${{ github.workspace }} + id: build + shell: bash + run: | + mkdir -p ${{ github.workspace }}/build + cd ${{ github.workspace }}/build + bazel run //:devbuild --cli_edition=enterprise + + bazel build //bazel/settings:tag + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files //bazel/settings:tag) + build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") + echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT" + + - name: Remove local Terraform registry # otherwise the local registry would be used instead of the public registry + if: inputs.providerVersion != '' + shell: bash + run: | + bazel build //bazel/settings:tag + repository_root=$(git rev-parse --show-toplevel) + out_rel=$(bazel cquery --output=files //bazel/settings:tag) + build_version=$(cat "$(realpath "${repository_root}/${out_rel}")") + + terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/" + rm -rf "${terraform_provider_dir}" + + - name: Login to AWS (IAM + Cluster role) + if: steps.determine.outputs.cloudProvider == 'aws' + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + with: + role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform + aws-region: eu-central-1 + # extend token expiry to 6 hours to ensure constellation can terminate + role-duration-seconds: 21600 + + - name: Login to Azure (IAM + Cluster service principal) + if: steps.determine.outputs.cloudProvider == 'azure' + uses: ./.github/actions/login_azure + with: + azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }} + + - name: Login to GCP (IAM + Cluster service account) + if: steps.determine.outputs.cloudProvider == 'gcp' + uses: ./.github/actions/login_gcp + with: + service_account: "terraform-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Set Kubernetes version + id: kubernetes + run: | + set -e + + # take the middle (2nd) supported Kubernetes version (default) + if [[ "${{ inputs.providerVersion }}" != "" ]]; then + cli_output=$(${{ github.workspace }}/release/constellation config kubernetes-versions) + else + cli_output=$(${{ github.workspace }}/build/constellation config kubernetes-versions) + fi + echo "version=$(echo "${cli_output}" | awk 'NR==3{print $1}')" | tee -a "${GITHUB_OUTPUT}" + + - name: Common CSP Terraform overrides + working-directory: ${{ github.workspace }} + shell: bash + run: | + mkdir -p ${{ github.workspace }}/cluster + cd ${{ github.workspace }}/cluster + if [[ "${{ inputs.providerVersion }}" == "" ]]; then + prefixed_version=${{ steps.build.outputs.build_version }} + else + prefixed_version="${{ inputs.providerVersion }}" + fi + version=${prefixed_version#v} # remove v prefix + + if [[ "${{ inputs.providerVersion }}" == "" ]]; then + iam_src="${{ github.workspace }}/terraform/infrastructure/iam/${{ steps.determine.outputs.cloudProvider }}" + infra_src="${{ github.workspace }}/terraform/infrastructure/${{ steps.determine.outputs.cloudProvider }}" + else + iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ steps.determine.outputs.cloudProvider }}" + infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ steps.determine.outputs.cloudProvider }}" + fi + + # by default use latest nightly image for devbuilds and release image otherwise + if [[ "${{ inputs.providerVersion }}" == "" ]]; then + if [[ "${{ inputs.image }}" == "" ]]; then + image_version="${{ steps.find-latest-image.outputs.image }}" + else + image_version="${{ inputs.image }}" + fi + else + if [[ "${{ inputs.image }}" == "" ]]; then + image_version="${prefixed_version}" + else + image_version="${{ inputs.image }}" + fi + fi + + kubernetes_version="${{ steps.kubernetes.outputs.version }}" + + cat > _override.tf <> _override.tf <> _override.tf <> _override.tf <> _override.tf <> /etc/hosts' + terraform init + if [[ "${{ inputs.attestationVariant }}" == "azure-sev-snp" ]]; then + timeout 1h terraform apply -target module.azure_iam -auto-approve + timeout 1h terraform apply -target module.azure_infrastructure -auto-approve + ${{ github.workspace }}/build/constellation maa-patch "$(terraform output -raw maa_url)" + timeout 1h terraform apply -target constellation_cluster.azure_example -auto-approve + else + timeout 1h terraform apply -auto-approve + fi + + - name: Cleanup Terraform Cluster on failure + # cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later + # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) + if: failure() && steps.apply_terraform.outcome != 'skipped' + working-directory: ${{ github.workspace }}/cluster + shell: bash + run: | + terraform init + terraform destroy -auto-approve -lock=false + + - name: Add Provider to local Terraform registry # needed if release version was used before + if: inputs.providerVersion != '' + working-directory: ${{ github.workspace }}/build + shell: bash + run: | + bazel run //:devbuild --cli_edition=enterprise + + - name: Update cluster configuration # for duplicate variable declaration, the last one is used + working-directory: ${{ github.workspace }}/cluster + shell: bash + run: | + cat >> _override.tf <> _override.tf <> _override.tf <> _override.tf < constellation-admin.conf + + if [[ -n "${MICROSERVICES}" ]]; then + MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}" + fi + if [[ -n "${KUBERNETES}" ]]; then + KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}" + fi + if [[ -n "${IMAGE}" ]]; then + IMAGE_FLAG="--target-image=${IMAGE}" + fi + + # cfg must be in same dir as KUBECONFIG + ${{ github.workspace }}/build/constellation config generate "${{ steps.determine.outputs.cloudProvider }}" --attestation ${{ inputs.attestationVariant}} + # make cfg valid with fake data + # IMPORTANT: zone needs to be correct because it is used to resolve the CSP image ref + if [[ "${{ steps.determine.outputs.cloudProvider }}" == "azure" ]]; then + location="${{ inputs.regionZone || 'northeurope' }}" + yq e ".provider.azure.location = \"${location}\"" -i constellation-conf.yaml + + yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml + yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml + yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml + yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml + fi + if [[ "${{ steps.determine.outputs.cloudProvider }}" == "gcp" ]]; then + zone="${{ inputs.regionZone || 'europe-west3-b' }}" + region=$(echo "${zone}" | rev | cut -c 2- | rev) + yq e ".provider.gcp.region = \"${region}\"" -i constellation-conf.yaml + yq e ".provider.gcp.zone = \"${zone}\"" -i constellation-conf.yaml + + yq e '.provider.gcp.project = "demo-gcp-project"' -i constellation-conf.yaml + yq e '.nodeGroups.control_plane_default.zone = "europe-west3-b"' -i constellation-conf.yaml + # Set the zone for worker_default node group to a fictional value + yq e '.nodeGroups.worker_default.zone = "europe-west3-b"' -i constellation-conf.yaml + yq e '.provider.gcp.serviceAccountKeyPath = "/path/to/your/service-account-key.json"' -i constellation-conf.yaml + fi + if [[ "${{ steps.determine.outputs.cloudProvider }}" == "aws" ]]; then + zone=${{ inputs.regionZone || 'us-east-2c' }} + region=$(echo "${zone}" | rev | cut -c 2- | rev) + yq e ".provider.aws.region = \"${region}\"" -i constellation-conf.yaml + yq e ".provider.aws.zone = \"${zone}\"" -i constellation-conf.yaml + + yq e '.provider.aws.iamProfileControlPlane = "demoControlPlaneIAMProfile"' -i constellation-conf.yaml + yq e '.provider.aws.iamProfileWorkerNodes = "demoWorkerNodesIAMProfile"' -i constellation-conf.yaml + yq e '.nodeGroups.control_plane_default.zone = "eu-central-1a"' -i constellation-conf.yaml + yq e '.nodeGroups.worker_default.zone = "eu-central-1a"' -i constellation-conf.yaml + fi + KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run --test_timeout=14400 //e2e/provider-upgrade:provider-upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --cli "${{ github.workspace }}/build/constellation" "$IMAGE_FLAG" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" + + - name: Destroy Terraform Cluster + # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) + if: always() && steps.apply_terraform.outcome != 'skipped' + working-directory: ${{ github.workspace }}/cluster + shell: bash + run: | + terraform init + terraform destroy -auto-approve -lock=false + + - name: Notify about failure + if: | + (failure() || cancelled()) && + github.ref == 'refs/heads/main' && + github.event_name == 'schedule' + continue-on-error: true + uses: ./.github/actions/notify_e2e_failure + with: + projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} + test: "terraform-provider-example" + refStream: ${{ inputs.ref}} + provider: ${{ steps.determine.outputs.cloudProvider }} + kubernetesVersion: ${{ steps.kubernetes.outputs.version }} + clusterCreation: "terraform" + attestationVariant: ${{ inputs.attestationVariant }} diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index e6a4977ee..4b8f5beb0 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -39,180 +39,98 @@ jobs: fail-fast: false max-parallel: 9 matrix: - include: - # - # Tests on ubuntu runner - # + test: + - "sonobuoy full" + - "verify" + - "recover" + - "lb" + - "autoscaling" + - "perf-bench" + - "malicious join" + attestationVariant: + - "gcp-sev-es" + - "gcp-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "aws-sev-snp" + kubernetes-version: ["v1.32"] + clusterCreation: ["cli"] + runner: ["ubuntu-24.04"] + include: # sonobuoy full test on all k8s versions - test: "sonobuoy full" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.31" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.31" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "aws" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - - - test: "sonobuoy full" - provider: "gcp" - kubernetes-version: "v1.27" - runner: "ubuntu-22.04" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.31" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "azure" - kubernetes-version: "v1.27" - runner: "ubuntu-22.04" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.31" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "aws" - kubernetes-version: "v1.27" - runner: "ubuntu-22.04" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.31" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "gcp" - kubernetes-version: "v1.26" - runner: "ubuntu-22.04" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.30" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "azure" - kubernetes-version: "v1.26" - runner: "ubuntu-22.04" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.30" + runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" - provider: "aws" - kubernetes-version: "v1.26" - runner: "ubuntu-22.04" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.30" + runner: "ubuntu-24.04" clusterCreation: "cli" - - # verify test on latest k8s version - - test: "verify" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "verify" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "verify" - provider: "aws" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - # recover test on latest k8s version - - test: "recover" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "recover" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "recover" - provider: "aws" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - # lb test on latest k8s version - - test: "lb" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "lb" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "lb" - provider: "aws" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - # autoscaling test on latest k8s version - - test: "autoscaling" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "autoscaling" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "autoscaling" - provider: "aws" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - # perf-bench test on latest k8s version, not supported on AWS - - test: "perf-bench" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - test: "perf-bench" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "cli" - - # self-managed infra test on latest k8s version - # runs Sonobuoy full test - test: "sonobuoy full" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "self-managed" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.30" + runner: "ubuntu-24.04" + clusterCreation: "cli" - test: "sonobuoy full" - provider: "azure" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "self-managed" - - test: "sonobuoy full" - provider: "aws" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" - clusterCreation: "self-managed" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.30" + runner: "ubuntu-24.04" + clusterCreation: "cli" # s3proxy test on latest k8s version - test: "s3proxy" - refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "ubuntu-22.04" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" + runner: "ubuntu-24.04" clusterCreation: "cli" # # Tests on macOS runner # # Skipping verify test on MacOS since the runner uses a different version of sed - # TODO(3u13r): Update verify test to work on MacOS runners + # TODO: Update verify test to work on MacOS runners # - test: "verify" - # provider: "azure" - # kubernetes-version: "v1.28" - # runner: "macos-12" + # attestationVariant: "azure-sev-snp" + # kubernetes-version: "v1.31" + # runner: "macos-latest" - test: "recover" - provider: "gcp" - kubernetes-version: "v1.28" - runner: "macos-12" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" + runner: "macos-latest" clusterCreation: "cli" runs-on: ${{ matrix.runner }} permissions: @@ -220,6 +138,7 @@ jobs: checks: write contents: read packages: write + actions: write steps: - name: Install the basics tools (macOS) if: runner.os == 'macOS' @@ -227,14 +146,23 @@ jobs: run: brew install coreutils kubectl bash - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 ref: ${{ inputs.ref || github.head_ref }} + - name: Split attestationVariant + id: split-attestationVariant + shell: bash + run: | + attestationVariant="${{ matrix.attestationVariant }}" + cloudProvider="${attestationVariant%%-*}" + + echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" + - name: Set up gcloud CLI (macOS) - if: matrix.provider == 'gcp' && runner.os == 'macOS' - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + if: steps.split-attestationVariant.outputs.provider == 'gcp' && runner.os == 'macOS' + uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 - name: Run E2E test id: e2e_test @@ -242,7 +170,8 @@ jobs: with: workerNodesCount: "2" controlNodesCount: "3" - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} cliVersion: "" kubernetesVersion: ${{ matrix.kubernetes-version }} osImage: "" @@ -255,7 +184,7 @@ jobs: gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" test: ${{ matrix.test }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io @@ -265,6 +194,7 @@ jobs: clusterCreation: ${{ matrix.clusterCreation }} s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }} s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() @@ -272,7 +202,7 @@ jobs: with: kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} clusterCreation: ${{ matrix.clusterCreation }} - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" @@ -280,17 +210,27 @@ jobs: if: always() uses: ./.github/actions/constellation_iam_destroy with: - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + e2e-upgrade: strategy: fail-fast: false max-parallel: 1 matrix: - fromVersion: ["v2.13.0"] - cloudProvider: ["gcp", "azure", "aws"] + fromVersion: ["v2.23.1"] + attestationVariant: ["gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] name: Run upgrade tests secrets: inherit permissions: @@ -298,10 +238,11 @@ jobs: contents: read checks: write packages: write + actions: write uses: ./.github/workflows/e2e-upgrade.yml with: fromVersion: ${{ matrix.fromVersion }} toImage: ${{ inputs.targetVersion }} - cloudProvider: ${{ matrix.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} nodeCount: '3:2' gitRef: ${{ inputs.ref || github.head_ref }} diff --git a/.github/workflows/e2e-test-self-managed.yml b/.github/workflows/e2e-test-self-managed.yml deleted file mode 100644 index fbdabe2f5..000000000 --- a/.github/workflows/e2e-test-self-managed.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: e2e test self managed infrastructure - -on: - workflow_dispatch: - inputs: - nodeCount: - description: "Number of nodes to use in the cluster. Given in format `:`." - default: "3:2" - type: string - cloudProvider: - description: "Which cloud provider to use." - type: choice - options: - - "gcp" - - "azure" - - "aws" - default: "azure" - required: true - runner: - description: "Architecture of the runner that executes the CLI" - type: choice - options: - - "ubuntu-22.04" - - "macos-12" - default: "ubuntu-22.04" - test: - description: "The test to run." - type: choice - options: - - "sonobuoy quick" - - "sonobuoy full" - - "autoscaling" - - "lb" - - "perf-bench" - - "verify" - - "recover" - - "malicious join" - - "nop" - required: true - kubernetesVersion: - description: "Kubernetes version to create the cluster from." - default: "1.27" - required: true - cliVersion: - description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." - type: string - default: "" - required: false - imageVersion: - description: "Full name of OS image (CSP independent image version UID). Leave empty for latest debug image on main." - type: string - default: "" - required: false - machineType: - description: "Override VM machine type. Leave as 'default' or empty to use the default VM type for the selected cloud provider." - type: string - default: "default" - required: false - regionZone: - description: "Region or zone to create the cluster in. Leave empty for default region/zone." - type: string - git-ref: - description: "Git ref to checkout." - type: string - default: "head" - required: false - -jobs: - e2e-test: - permissions: - id-token: write - checks: write - contents: read - packages: write - secrets: inherit - uses: ./.github/workflows/e2e-test.yml - with: - nodeCount: ${{ inputs.nodeCount }} - cloudProvider: ${{ inputs.cloudProvider }} - runner: ${{ inputs.runner }} - test: ${{ inputs.test }} - kubernetesVersion: ${{ inputs.kubernetesVersion }} - cliVersion: ${{ inputs.cliVersion }} - imageVersion: ${{ inputs.imageVersion }} - machineType: ${{ inputs.machineType }} - regionZone: ${{ inputs.regionZone }} - git-ref: ${{ inputs.git-ref }} - clusterCreation: "self-managed" diff --git a/.github/workflows/e2e-test-stackit.yml b/.github/workflows/e2e-test-stackit.yml new file mode 100644 index 000000000..1977d09cc --- /dev/null +++ b/.github/workflows/e2e-test-stackit.yml @@ -0,0 +1,153 @@ +name: e2e test STACKIT + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # Every day at midnight. + +jobs: + find-latest-image: + name: Find latest image + runs-on: ubuntu-24.04 + permissions: + id-token: write + contents: read + outputs: + image-release-stable: ${{ steps.relabel-output.outputs.image-release-stable }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Select relevant image + id: select-image-action + uses: ./.github/actions/select_image + with: + osImage: "ref/release/stream/stable/?" + + - name: Relabel output + id: relabel-output + shell: bash + run: | + ref=$(echo 'ref/release/stream/stable/?' | cut -d/ -f2) + stream=$(echo 'ref/release/stream/stable/?' | cut -d/ -f4) + + echo "image-$ref-$stream=${{ steps.select-image-action.outputs.osImage }}" | tee -a "$GITHUB_OUTPUT" + + e2e-stackit: + strategy: + fail-fast: false + max-parallel: 6 + matrix: + kubernetesVersion: [ "1.29", "1.30", "1.31" ] + clusterCreation: [ "cli", "terraform" ] + test: [ "sonobuoy quick" ] + runs-on: ubuntu-24.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + actions: write + needs: [find-latest-image] + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Setup bazel + uses: ./.github/actions/setup_bazel_nix + with: + nixTools: terraform + + - name: Run E2E test + id: e2e_test + uses: ./.github/actions/e2e_test + with: + workerNodesCount: "1" + controlNodesCount: "1" + cloudProvider: stackit + attestationVariant: qemu-vtpm + osImage: ${{ needs.find-latest-image.outputs.image-release-stable }} + isDebugImage: false + cliVersion: ${{ needs.find-latest-image.outputs.image-release-stable || '' }} + kubernetesVersion: ${{ matrix.kubernetesVersion }} + awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }} + awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} + awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} + gcpProject: constellation-e2e + gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" + gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + test: ${{ matrix.test }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} + azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + registry: ghcr.io + githubToken: ${{ secrets.GITHUB_TOKEN }} + cosignPassword: ${{ secrets.COSIGN_PASSWORD }} + cosignPrivateKey: ${{ secrets.COSIGN_PRIVATE_KEY }} + fetchMeasurements: false + clusterCreation: ${{ matrix.clusterCreation }} + s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }} + s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }} + stackitUat: ${{ secrets.STACKIT_CI_UAT }} + stackitProjectID: ${{ secrets.STACKIT_CI_PROJECT_ID }} + + - name: Always terminate cluster + if: always() + uses: ./.github/actions/constellation_destroy + with: + kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} + clusterCreation: ${{ matrix.clusterCreation }} + cloudProvider: stackit + azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} + gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Always delete IAM configuration + if: always() + uses: ./.github/actions/constellation_iam_destroy + with: + cloudProvider: stackit + azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Notify about failure + if: | + failure() && + github.ref == 'refs/heads/main' && + github.event_name == 'schedule' + continue-on-error: true + uses: ./.github/actions/notify_e2e_failure + with: + projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} + refStream: "ref/release/stream/stable/?" + test: ${{ matrix.test }} + kubernetesVersion: ${{ matrix.kubernetesVersion }} + provider: stackit + attestationVariant: qemu-vtpm + clusterCreation: ${{ matrix.clusterCreation }} + + - name: Notify STACKIT + if: | + failure() && + github.ref == 'refs/heads/main' && + github.event_name == 'schedule' + continue-on-error: true + uses: ./.github/actions/notify_stackit + with: + slackToken: ${{ secrets.SLACK_TOKEN }} diff --git a/.github/workflows/e2e-test-terraform-provider.yml b/.github/workflows/e2e-test-terraform-provider.yml index 7a3b80777..c7aa5a0e5 100644 --- a/.github/workflows/e2e-test-terraform-provider.yml +++ b/.github/workflows/e2e-test-terraform-provider.yml @@ -7,22 +7,24 @@ on: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: choice options: - - "gcp" - - "azure" - - "aws" - default: "azure" + - "aws-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "gcp-sev-es" + - "gcp-sev-snp" + default: "azure-sev-snp" required: true runner: description: "Architecture of the runner that executes the CLI" type: choice options: - - "ubuntu-22.04" - - "macos-12" - default: "ubuntu-22.04" + - "ubuntu-24.04" + - "macos-latest" + default: "ubuntu-24.04" test: description: "The test to run." type: choice @@ -39,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.27" required: true releaseVersion: description: "Version of a released provider to download. Leave empty to build the provider from the checked out ref." @@ -76,7 +77,7 @@ jobs: uses: ./.github/workflows/e2e-test.yml with: nodeCount: ${{ inputs.nodeCount }} - cloudProvider: ${{ inputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} runner: ${{ inputs.runner }} test: ${{ inputs.test }} kubernetesVersion: ${{ inputs.kubernetesVersion }} diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 49541ad11..dbc33d9fd 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -10,9 +10,9 @@ jobs: strategy: fail-fast: false matrix: - refStream: ["ref/main/stream/nightly/?","ref/main/stream/debug/?", "ref/release/stream/stable/?"] + refStream: ["ref/main/stream/nightly/?", "ref/main/stream/debug/?", "ref/release/stream/stable/?"] name: Find latest image - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -22,7 +22,7 @@ jobs: image-main-nightly: ${{ steps.relabel-output.outputs.image-main-nightly }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} @@ -51,192 +51,261 @@ jobs: # Tests on main-debug refStream # - # sonobuoy full test on all k8s versions - - test: "sonobuoy full" + # Emergency SSH test on latest k8s version + - test: "emergency ssh" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - - test: "sonobuoy full" + - test: "emergency ssh" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - - test: "sonobuoy full" + - test: "emergency ssh" refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.28" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "emergency ssh" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "emergency ssh" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" + # Sonobuoy full test on latest k8s version - test: "sonobuoy full" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.27" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "sonobuoy full" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.27" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "sonobuoy full" refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.27" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "sonobuoy full" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "sonobuoy full" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - - test: "sonobuoy full" + # Sonobuoy conformance test + - test: "sonobuoy conformance" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.26" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - - test: "sonobuoy full" + + # Sonobuoy quick test on all but the latest k8s versions + - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.26" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.31" clusterCreation: "cli" - - test: "sonobuoy full" + - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.26" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.31" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.31" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.31" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.31" + clusterCreation: "cli" + + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.30" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.30" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.30" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.30" + clusterCreation: "cli" + - test: "sonobuoy quick" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.30" clusterCreation: "cli" # verify test on latest k8s version - test: "verify" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "verify" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "verify" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" azureSNPEnforcementPolicy: "equal" # This run checks for unknown ID Key disgests. clusterCreation: "cli" - test: "verify" - provider: "aws" refStream: "ref/main/stream/debug/?" - kubernetes-version: "v1.28" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "verify" + attestationVariant: "aws-sev-snp" + refStream: "ref/main/stream/debug/?" + kubernetes-version: "v1.32" clusterCreation: "cli" # recover test on latest k8s version - test: "recover" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "recover" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "recover" refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.28" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "recover" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "recover" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" # lb test on latest k8s version - test: "lb" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "lb" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "lb" refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.28" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "lb" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "lb" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" # autoscaling test on latest k8s version - test: "autoscaling" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "autoscaling" refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "autoscaling" refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.28" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "autoscaling" + refStream: "ref/main/stream/debug/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" + clusterCreation: "cli" + - test: "autoscaling" + refStream: "ref/main/stream/debug/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - # perf-bench test on latest k8s version, not supported on AWS + # perf-bench test on latest k8s version - test: "perf-bench" - refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + refStream: "ref/main/stream/nightly/?" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" - test: "perf-bench" - refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + refStream: "ref/main/stream/nightly/?" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - - # malicious join test on latest k8s version - - test: "malicious join" - refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + - test: "perf-bench" + refStream: "ref/main/stream/nightly/?" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - - test: "malicious join" - refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" + - test: "perf-bench" + refStream: "ref/main/stream/nightly/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.32" clusterCreation: "cli" - - test: "malicious join" - refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.28" + - test: "perf-bench" + refStream: "ref/main/stream/nightly/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.32" clusterCreation: "cli" - # self-managed infra test on latest k8s version - # with Sonobuoy full - - test: "sonobuoy full" - refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" - clusterCreation: "self-managed" - - test: "sonobuoy full" - refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" - clusterCreation: "self-managed" - - test: "sonobuoy full" - provider: "aws" - refStream: "ref/main/stream/debug/?" - kubernetes-version: "v1.28" - clusterCreation: "self-managed" - - - test: "sonobuoy full" - refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" - clusterCreation: "terraform" - - test: "sonobuoy full" - refStream: "ref/main/stream/debug/?" - provider: "azure" - kubernetes-version: "v1.28" - clusterCreation: "terraform" - - test: "sonobuoy full" - refStream: "ref/main/stream/debug/?" - provider: "aws" - kubernetes-version: "v1.28" - clusterCreation: "terraform" - # s3proxy test on latest k8s version - test: "s3proxy" refStream: "ref/main/stream/debug/?" - provider: "gcp" - kubernetes-version: "v1.28" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.32" clusterCreation: "cli" # @@ -246,42 +315,63 @@ jobs: # verify test on default k8s version - test: "verify" refStream: "ref/release/stream/stable/?" - provider: "gcp" - kubernetes-version: "v1.27" + attestationVariant: "gcp-sev-es" + kubernetes-version: "v1.31" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" - provider: "azure" - kubernetes-version: "v1.27" + attestationVariant: "gcp-sev-snp" + kubernetes-version: "v1.31" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" - provider: "aws" - kubernetes-version: "v1.27" + attestationVariant: "azure-sev-snp" + kubernetes-version: "v1.31" + clusterCreation: "cli" + - test: "verify" + refStream: "ref/release/stream/stable/?" + attestationVariant: "azure-tdx" + kubernetes-version: "v1.31" + clusterCreation: "cli" + - test: "verify" + refStream: "ref/release/stream/stable/?" + attestationVariant: "aws-sev-snp" + kubernetes-version: "v1.31" clusterCreation: "cli" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write checks: write contents: read packages: write + actions: write needs: [find-latest-image] steps: - name: Check out repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + - name: Split attestationVariant + id: split-attestationVariant + shell: bash + run: | + attestationVariant="${{ matrix.attestationVariant }}" + cloudProvider="${attestationVariant%%-*}" + + echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" + - name: Run E2E test id: e2e_test uses: ./.github/actions/e2e_test with: workerNodesCount: "2" controlNodesCount: "3" - cloudProvider: ${{ matrix.provider }} - osImage: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} + osImage: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || matrix.refStream == 'ref/main/stream/nightly/?' && needs.find-latest-image.outputs.image-main-nightly || needs.find-latest-image.outputs.image-main-debug }} isDebugImage: ${{ matrix.refStream == 'ref/main/stream/debug/?' }} cliVersion: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || '' }} kubernetesVersion: ${{ matrix.kubernetes-version }} @@ -293,7 +383,7 @@ jobs: gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" test: ${{ matrix.test }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io @@ -305,6 +395,7 @@ jobs: clusterCreation: ${{ matrix.clusterCreation }} s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }} s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() @@ -312,7 +403,7 @@ jobs: with: kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} clusterCreation: ${{ matrix.clusterCreation }} - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" @@ -320,10 +411,20 @@ jobs: if: always() uses: ./.github/actions/constellation_iam_destroy with: - cloudProvider: ${{ matrix.provider }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Notify about failure if: | failure() && @@ -336,7 +437,8 @@ jobs: refStream: ${{ matrix.refStream }} test: ${{ matrix.test }} kubernetesVersion: ${{ matrix.kubernetes-version }} - provider: ${{ matrix.provider }} + provider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} clusterCreation: ${{ matrix.clusterCreation }} e2e-upgrade: @@ -344,8 +446,8 @@ jobs: fail-fast: false max-parallel: 1 matrix: - fromVersion: ["v2.13.0"] - cloudProvider: ["gcp", "azure", "aws"] + fromVersion: ["v2.23.1"] + attestationVariant: ["gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] name: Run upgrade tests secrets: inherit permissions: @@ -353,16 +455,17 @@ jobs: checks: write contents: read packages: write + actions: write uses: ./.github/workflows/e2e-upgrade.yml with: fromVersion: ${{ matrix.fromVersion }} - cloudProvider: ${{ matrix.cloudProvider }} + attestationVariant: ${{ matrix.attestationVariant }} nodeCount: '3:2' scheduled: ${{ github.event_name == 'schedule' }} e2e-mini: name: Run miniconstellation E2E test - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 environment: e2e permissions: id-token: write @@ -371,12 +474,12 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Azure login OIDC - uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.7 + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0 with: client-id: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} @@ -388,7 +491,7 @@ jobs: azureClientID: ${{ secrets.AZURE_E2E_MINI_CLIENT_ID }} azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureTenantID: ${{ secrets.AZURE_TENANT_ID }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureIAMCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io githubToken: ${{ secrets.GITHUB_TOKEN }} @@ -403,6 +506,7 @@ jobs: projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} test: "MiniConstellation" provider: "QEMU" + attestationVariant: "qemu-vtpm" e2e-windows: name: Run Windows E2E test @@ -410,7 +514,23 @@ jobs: id-token: write contents: read packages: write + checks: write secrets: inherit uses: ./.github/workflows/e2e-windows.yml with: scheduled: ${{ github.event_name == 'schedule' }} + + e2e-terraform-provider-example: + name: Run Terraform provider example E2E test + strategy: + fail-fast: false + matrix: + attestationVariant: ["gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] + permissions: + id-token: write + contents: read + packages: write + secrets: inherit + uses: ./.github/workflows/e2e-test-provider-example.yml + with: + attestationVariant: ${{ matrix.attestationVariant }} diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c345926e2..46efa290d 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -7,28 +7,32 @@ on: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: choice options: - - "gcp" - - "azure" - - "aws" - default: "azure" + - "gcp-sev-es" + - "gcp-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "aws-sev-snp" + - "stackit-qemu-vtpm" + default: "azure-sev-snp" required: true runner: description: "Architecture of the runner that executes the CLI" type: choice options: - - "ubuntu-22.04" - - "macos-12" - default: "ubuntu-22.04" + - "ubuntu-24.04" + - "macos-latest" + default: "ubuntu-24.04" test: - description: "The test to run." + description: "The test to run. The conformance test is only supported for clusterCreation=cli." type: choice options: - "sonobuoy quick" - "sonobuoy full" + - "sonobuoy conformance" - "autoscaling" - "lb" - "perf-bench" @@ -36,11 +40,12 @@ on: - "recover" - "malicious join" - "s3proxy" + - "emergency ssh" - "nop" required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.27" + default: "1.30" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." @@ -71,8 +76,8 @@ on: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: string required: true runner: @@ -80,7 +85,7 @@ on: type: string required: true test: - description: "The test to run." + description: "The test to run. The conformance test is only supported for clusterCreation=cli." type: string required: true kubernetesVersion: @@ -113,7 +118,7 @@ on: type: boolean default: false clusterCreation: - description: "How to create infrastructure for the e2e test. One of [cli, self-managed, terraform]." + description: "How to create infrastructure for the e2e test. One of [cli, terraform]." type: string default: "cli" marketplaceImageVersion: @@ -124,15 +129,17 @@ on: type: boolean jobs: - split-nodeCount: - name: Split nodeCount - runs-on: ubuntu-22.04 + generate-input-parameters: + name: Generate input parameters + runs-on: ubuntu-24.04 permissions: id-token: write contents: read outputs: workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }} controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} + attestationVariant: ${{ steps.split-attestationVariant.outputs.attestationVariant }} steps: - name: Split nodeCount id: split-nodeCount @@ -150,9 +157,24 @@ jobs: echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT" echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT" + - name: Split attestationVariant + id: split-attestationVariant + shell: bash + run: | + attestationVariant="${{ inputs.attestationVariant }}" + cloudProvider="${attestationVariant%%-*}" + + # special case for STACKIT, as there's no special attestation variant for it + if [[ "${cloudProvider}" == "stackit" ]]; then + attestationVariant="qemu-vtpm" + fi + + echo "attestationVariant=${attestationVariant}" | tee -a "$GITHUB_OUTPUT" + echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" + find-latest-image: name: Select image - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -162,13 +184,13 @@ jobs: steps: - name: Checkout head if: inputs.git-ref == 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.git-ref != 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git-ref }} @@ -188,7 +210,8 @@ jobs: checks: write contents: read packages: write - needs: [find-latest-image, split-nodeCount] + actions: write + needs: [find-latest-image, generate-input-parameters] if: always() && !cancelled() steps: - name: Install basic tools (macOS) @@ -198,27 +221,28 @@ jobs: - name: Checkout head if: inputs.git-ref == 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.git-ref != 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.git-ref }} - name: Set up gcloud CLI (macOS) - if: inputs.cloudProvider == 'gcp' && runner.os == 'macOS' - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp' && runner.os == 'macOS' + uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 - name: Run manual E2E test id: e2e_test uses: ./.github/actions/e2e_test with: - workerNodesCount: ${{ needs.split-nodeCount.outputs.workerNodes }} - controlNodesCount: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }} - cloudProvider: ${{ inputs.cloudProvider }} + workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }} + controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }} + cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} + attestationVariant: ${{ needs.generate-input-parameters.outputs.attestationVariant }} machineType: ${{ inputs.machineType }} regionZone: ${{ inputs.regionZone }} gcpProject: constellation-e2e @@ -232,7 +256,7 @@ jobs: osImage: ${{ needs.find-latest-image.outputs.image }} cliVersion: ${{ inputs.cliVersion }} isDebugImage: ${{ needs.find-latest-image.outputs.isDebugImage }} - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io @@ -246,6 +270,10 @@ jobs: s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }} force: ${{ inputs.force }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + openStackCloudsYaml: ${{ secrets.STACKIT_CI_CLOUDS_YAML }} + stackitUat: ${{ secrets.STACKIT_CI_UAT }} + stackitProjectID: ${{ secrets.STACKIT_CI_PROJECT_ID }} - name: Always terminate cluster if: always() @@ -253,7 +281,7 @@ jobs: with: kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} clusterCreation: ${{ inputs.clusterCreation }} - cloudProvider: ${{ inputs.cloudProvider }} + cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" @@ -261,6 +289,16 @@ jobs: if: always() uses: ./.github/actions/constellation_iam_destroy with: - cloudProvider: ${{ inputs.cloudProvider }} + cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index e7374414e..cd7c6bc1b 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -3,14 +3,17 @@ name: e2e test upgrade on: workflow_dispatch: inputs: - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: choice options: - - "gcp" - - "azure" - - "aws" - default: "azure" + - "aws-sev-snp" + - "azure-sev-snp" + - "azure-tdx" + - "gcp-sev-es" + - "gcp-sev-snp" + default: "azure-sev-snp" + required: true nodeCount: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" @@ -19,6 +22,10 @@ on: description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'. type: string required: true + fromKubernetes: + description: Kubernetes version for the origin cluster, empty for origin target's default version. + type: string + required: false gitRef: description: Ref to build upgrading CLI on, empty for HEAD. type: string @@ -29,11 +36,11 @@ on: type: string required: false toKubernetes: - description: Kubernetes version to target for the upgrade, empty for target's default version. + description: Kubernetes version to target for the upgrade, empty for upgrade target's default version. type: string required: false toMicroservices: - description: Microservice version to target for the upgrade, empty for target's default version. + description: Microservice version to target for the upgrade, empty for upgrade target's default version. type: string required: false simulatedTargetVersion: @@ -45,8 +52,8 @@ on: type: string workflow_call: inputs: - cloudProvider: - description: "Which cloud provider to use." + attestationVariant: + description: "Which attestation variant to use." type: string required: true nodeCount: @@ -57,6 +64,10 @@ on: description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'. type: string required: true + fromKubernetes: + description: Kubernetes version for the origin cluster, empty for origin target's default version. + type: string + required: false gitRef: description: Ref to build upgrading CLI on. type: string @@ -85,15 +96,16 @@ on: required: false jobs: - split-nodeCount: - name: Split nodeCount - runs-on: ubuntu-22.04 + generate-input-parameters: + name: Generate input parameters + runs-on: ubuntu-24.04 permissions: id-token: write contents: read outputs: workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }} controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }} + cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} steps: - name: Split nodeCount id: split-nodeCount @@ -111,36 +123,158 @@ jobs: echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT" echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT" - e2e-upgrade: - runs-on: ubuntu-22.04 + - name: Split attestationVariant + id: split-attestationVariant + shell: bash + run: | + attestationVariant="${{ inputs.attestationVariant }}" + cloudProvider="${attestationVariant%%-*}" + + echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" + + create-cluster: + name: Create upgrade origin version cluster + runs-on: ubuntu-24.04 permissions: id-token: write checks: write contents: read packages: write - needs: [split-nodeCount] + needs: [generate-input-parameters] + outputs: + kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} + e2e-name-prefix: ${{ steps.e2e_test.outputs.namePrefix }} steps: - name: Checkout if: inputs.gitRef == 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.gitRef != 'head' - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 ref: ${{ inputs.gitRef }} - uses: ./.github/actions/setup_bazel_nix + + - name: Create cluster with 'fromVersion' CLI. + id: e2e_test + uses: ./.github/actions/e2e_test with: - useCache: "true" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }} + controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }} + cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} + osImage: ${{ inputs.fromVersion }} + isDebugImage: "false" + cliVersion: ${{ inputs.fromVersion }} + kubernetesVersion: ${{ inputs.fromKubernetes }} + regionZone: ${{ inputs.regionZone }} + gcpProject: constellation-e2e + gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" + gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + test: "upgrade" + azureSubscriptionID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} + azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + registry: ghcr.io + githubToken: ${{ secrets.GITHUB_TOKEN }} + awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }} + awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} + awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} + clusterCreation: "cli" + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Remove Terraform plugin cache + if: always() + run: | + rm -rf constellation-terraform/.terraform + rm -rf constellation-iam-terraform/.terraform + + - name: Upload Working Directory + if: always() + uses: ./.github/actions/artifact_upload + with: + name: constellation-pre-test-${{ inputs.attestationVariant }} + path: > + ${{ steps.e2e_test.outputs.kubeconfig }} + constellation-terraform + constellation-iam-terraform + constellation-conf.yaml + constellation-state.yaml + constellation-mastersecret.json + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Upload SA Key + if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp' + uses: ./.github/actions/artifact_upload + with: + name: sa-key-${{ inputs.attestationVariant }} + path: > + gcpServiceAccountKey.json + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + e2e-upgrade: + name: Run upgrade test + runs-on: ubuntu-24.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + needs: + - generate-input-parameters + - create-cluster + steps: + - name: Checkout + if: inputs.gitRef == 'head' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Checkout ref + if: inputs.gitRef != 'head' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ inputs.gitRef }} + + - name: Setup Bazel & Nix + uses: ./.github/actions/setup_bazel_nix + + - name: Log in to the Container registry + uses: ./.github/actions/container_registry_login + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # applying the version manipulation here so that the upgrade test tool is also on the simulated target version + - name: Simulate patch upgrade + if: inputs.simulatedTargetVersion != '' + run: | + echo ${{ inputs.simulatedTargetVersion }} > version.txt + + - name: Build CLI + uses: ./.github/actions/build_cli + with: + enterpriseCLI: true + outputPath: "build/constellation" + push: true + + - name: Upload CLI binary # is needed for the cleanup step + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: constellation-upgrade-${{ inputs.attestationVariant }} + path: build/constellation - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead aws-region: eu-central-1 @@ -154,52 +288,15 @@ jobs: ref: main stream: nightly - - name: Simulate patch upgrade - if: inputs.simulatedTargetVersion != '' - run: | - echo ${{ inputs.simulatedTargetVersion }} > version.txt - - - name: Create cluster with 'fromVersion' CLI. - id: e2e_test - uses: ./.github/actions/e2e_test - with: - workerNodesCount: ${{ needs.split-nodeCount.outputs.workerNodes }} - controlNodesCount: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }} - cloudProvider: ${{ inputs.cloudProvider }} - osImage: ${{ inputs.fromVersion }} - isDebugImage: "false" - cliVersion: ${{ inputs.fromVersion }} - regionZone: ${{ inputs.regionZone }} - gcpProject: constellation-e2e - gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" - gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" - test: "upgrade" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} - azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} - registry: ghcr.io - githubToken: ${{ secrets.GITHUB_TOKEN }} - awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }} - awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} - awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} - clusterCreation: "cli" - - - name: Build CLI - uses: ./.github/actions/build_cli - with: - enterpriseCLI: true - outputPath: "build/constellation" - push: true - - name: Login to GCP (IAM service account) - if: inputs.cloudProvider == 'gcp' + if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp' uses: ./.github/actions/login_gcp with: service_account: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" - name: Login to AWS (IAM role) - if: inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + if: needs.generate-input-parameters.outputs.cloudProvider == 'aws' + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM aws-region: eu-central-1 @@ -207,11 +304,32 @@ jobs: role-duration-seconds: 21600 - name: Login to Azure (IAM service principal) - if: inputs.cloudProvider == 'azure' + if: needs.generate-input-parameters.outputs.cloudProvider == 'azure' uses: ./.github/actions/login_azure with: azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + + - name: Download Working Directory (Pre-test) + uses: ./.github/actions/artifact_download + with: + name: constellation-pre-test-${{ inputs.attestationVariant }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Download SA Key + if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp' + uses: ./.github/actions/artifact_download + with: + name: sa-key-${{ inputs.attestationVariant }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Make Constellation executable and add to PATH + if: always() + run: | + chmod +x build/constellation + export PATH="$PATH:build" + echo "build" >> "$GITHUB_PATH" + - name: Migrate config id: constellation-config-migrate run: | @@ -222,14 +340,14 @@ jobs: uses: ./.github/actions/constellation_iam_upgrade - name: Login to GCP (Cluster service account) - if: always() && inputs.cloudProvider == 'gcp' + if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp' uses: ./.github/actions/login_gcp with: service_account: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" - name: Login to AWS (Cluster role) - if: always() && inputs.cloudProvider == 'aws' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'aws' + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster aws-region: eu-central-1 @@ -237,37 +355,105 @@ jobs: role-duration-seconds: 21600 - name: Login to Azure (Cluster service principal) - if: always() && inputs.cloudProvider == 'azure' + if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'azure' uses: ./.github/actions/login_azure with: azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} - name: Run upgrade test env: - KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }} + KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }} IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }} KUBERNETES: ${{ inputs.toKubernetes }} MICROSERVICES: ${{ inputs.toMicroservices }} - WORKERNODES: ${{ needs.split-nodeCount.outputs.workerNodes }} - CONTROLNODES: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }} + WORKERNODES: ${{ needs.generate-input-parameters.outputs.workerNodes }} + CONTROLNODES: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }} run: | echo "Image target: $IMAGE" echo "K8s target: $KUBERNETES" echo "Microservice target: $MICROSERVICES" - if [[ -n ${MICROSERVICES} ]]; then - MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES" - fi - if [[ -n ${KUBERNETES} ]]; then - KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES" - fi + sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' + CLI=$(realpath ./build/constellation) + bazel run --test_timeout=14400 //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" --target-kubernetes "$KUBERNETES" --target-microservices "$MICROSERVICES" --cli "$CLI" - bazel run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" + - name: Remove Terraform plugin cache + if: always() + run: | + rm -rf constellation-terraform/.terraform + rm -rf constellation-iam-terraform/.terraform + + - name: Upload Working Directory + if: always() + uses: ./.github/actions/artifact_upload + with: + name: constellation-post-test-${{ inputs.attestationVariant }} + path: | + ${{ needs.create-cluster.outputs.kubeconfig }} + constellation-terraform + constellation-iam-terraform + constellation-conf.yaml + constellation-state.yaml + constellation-mastersecret.json + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + clean-up: + name: Clean up resources + runs-on: ubuntu-24.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + actions: write + if: always() + needs: [generate-input-parameters, create-cluster, e2e-upgrade] + steps: + - name: Checkout + if: inputs.gitRef == 'head' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Checkout ref + if: inputs.gitRef != 'head' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + ref: ${{ inputs.gitRef }} + + - name: Download CLI + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: constellation-upgrade-${{ inputs.attestationVariant }} + path: build + + - name: Download Working Directory (Pre-test) + if: always() && needs.e2e-upgrade.result != 'success' + uses: ./.github/actions/artifact_download + with: + name: constellation-pre-test-${{ inputs.attestationVariant }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Download Working Directory (Post-test) + if: always() && needs.e2e-upgrade.result == 'success' + uses: ./.github/actions/artifact_download + with: + name: constellation-post-test-${{ inputs.attestationVariant }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Make Constellation executable and add to PATH + if: always() + run: | + chmod +x build/constellation + export PATH="$PATH:build" + echo "build" >> "$GITHUB_PATH" - name: Always fetch logs if: always() env: - KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }} + KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }} run: | kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs @@ -275,21 +461,33 @@ jobs: - name: Always upload logs if: always() - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: ./.github/actions/artifact_upload with: - name: upgrade-logs - path: | + name: upgrade-logs-${{ inputs.attestationVariant }} + path: > node-operator.logs node-maintenance-operator.logs constellation-version.yaml + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Prepare terraform state artifact upload + if: always() + shell: bash + run: | + mkdir -p to-zip + cp -r constellation-terraform to-zip + rm -f to-zip/constellation-terraform/plan.zip + rm -rf to-zip/constellation-terraform/.terraform + cp -r constellation-iam-terraform to-zip + rm -rf to-zip/constellation-iam-terraform/.terraform - name: Always terminate cluster if: always() uses: ./.github/actions/constellation_destroy with: - kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} clusterCreation: "cli" - cloudProvider: ${{ inputs.cloudProvider }} + kubeconfig: ${{ needs.create-cluster.outputs.kubeconfig }} + cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" @@ -297,13 +495,24 @@ jobs: if: always() uses: ./.github/actions/constellation_iam_destroy with: - cloudProvider: ${{ inputs.cloudProvider }} + cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" + - name: Update tfstate + if: always() + env: + GH_TOKEN: ${{ github.token }} + uses: ./.github/actions/update_tfstate + with: + name: terraform-state-${{ needs.create-cluster.outputs.e2e-name-prefix }} + runID: ${{ github.run_id }} + encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + - name: Notify about failure if: | - failure() && + always() && + ( needs.create-cluster.result != 'success' || needs.e2e-upgrade.result != 'success' ) && github.ref == 'refs/heads/main' && inputs.scheduled continue-on-error: true @@ -311,4 +520,5 @@ jobs: with: projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} test: "upgrade" - provider: ${{ inputs.cloudProvider }} + provider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} + attestationVariant: ${{ inputs.attestationVariant }} diff --git a/.github/workflows/e2e-windows.yml b/.github/workflows/e2e-windows.yml index 5cd322c5e..ad4acb877 100644 --- a/.github/workflows/e2e-windows.yml +++ b/.github/workflows/e2e-windows.yml @@ -13,18 +13,27 @@ on: jobs: build-cli: name: Build Windows CLI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + permissions: + id-token: write + checks: write + contents: read + packages: write steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Setup bazel uses: ./.github/actions/setup_bazel_nix + + - name: Log in to the Container registry + uses: ./.github/actions/container_registry_login with: - useCache: "true" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build CLI uses: ./.github/actions/build_cli @@ -32,33 +41,35 @@ jobs: targetOS: "windows" targetArch: "amd64" enterpriseCLI: true + outputPath: "build/constellation" + push: true - name: Upload CLI artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - path: "bazel-bin/cli/cli_enterprise_windows_amd64" + path: build/constellation.exe name: "constell-exe" e2e-test: name: E2E Test Windows - runs-on: windows-2022 + runs-on: windows-2025 needs: build-cli steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Download CLI artifact - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: "constell-exe" - name: Check CLI version shell: pwsh run: | - Move-Item -Path .\cli_enterprise_windows_amd64 -Destination .\constellation.exe .\constellation.exe version + Add-Content -Path $env:windir\System32\drivers\etc\hosts -Value "`n127.0.0.1`tlicense.confidential.cloud" -Force - name: Login to Azure (IAM service principal) uses: ./.github/actions/login_azure @@ -66,25 +77,24 @@ jobs: azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} - name: Create IAM configuration + id: iam-create shell: pwsh run: | - .\constellation.exe config generate azure - .\constellation.exe iam create azure --region=westus --resourceGroup=e2eWindoewsRG --servicePrincipal=e2eWindoewsSP --update-config --debug -y + $uid = Get-Random -Minimum 1000 -Maximum 9999 + $rgName = "e2e-win-${{ github.run_id }}-${{ github.run_attempt }}-$uid" + "rgName=$($rgName)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + .\constellation.exe config generate azure -t "workflow=${{ github.run_id }}" + .\constellation.exe iam create azure --subscriptionID=${{ secrets.AZURE_SUBSCRIPTION_ID }} --region=westus --resourceGroup=$rgName-rg --servicePrincipal=$rgName-sp --update-config --debug -y - name: Login to Azure (Cluster service principal) uses: ./.github/actions/login_azure with: azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} - - name: Create cluster + - name: Apply config shell: pwsh run: | - .\constellation.exe create --debug -y - - - name: Initialize cluster - shell: pwsh - run: | - .\constellation.exe apply --debug + .\constellation.exe apply --debug -y - name: Liveness probe shell: pwsh @@ -100,24 +110,31 @@ jobs: Write-Host "Retry ${retryCount}: Checking node status..." $nodesOutput = & kubectl get nodes --kubeconfig "$PWD\constellation-admin.conf" + $status = $? - $lines = $nodesOutput -split "`r?`n" | Select-Object -Skip 1 + $nodesOutput - $allNodesReady = $true + if ($status) { + $lines = $nodesOutput -split "`r?`n" | Select-Object -Skip 1 - foreach ($line in $lines) { - $columns = $line -split '\s+' | Where-Object { $_ -ne '' } + if ($lines.count -eq 4) { + $allNodesReady = $true - $nodeName = $columns[0] - $status = $columns[1] + foreach ($line in $lines) { + $columns = $line -split '\s+' | Where-Object { $_ -ne '' } - if ($status -ne "Ready") { - Write-Host "Node $nodeName is not ready!" - $allNodesReady = $false + $nodeName = $columns[0] + $status = $columns[1] + + if ($status -ne "Ready") { + Write-Host "Node $nodeName is not ready!" + $allNodesReady = $false + } + } } } - if (-not $allNodesReady) { + if (-not $allNodesReady -and $retryCount -lt $maxRetries) { Write-Host "Retrying in $retryIntervalSeconds seconds..." Start-Sleep -Seconds $retryIntervalSeconds } @@ -132,6 +149,7 @@ jobs: } - name: Terminate cluster + id: terminate-cluster if: always() shell: pwsh run: | @@ -144,14 +162,23 @@ jobs: azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} - name: Delete IAM configuration + id: delete-iam if: always() shell: pwsh run: | .\constellation.exe iam destroy --debug -y + - name: Clean up after failure + # run on a cleanup failure or if cancelled + if: (failure() && (steps.terminate-cluster.conclusion == 'failure' || steps.delete-iam.conclusion == 'failure')) || cancelled() + shell: pwsh + run: | + az group delete --name ${{ steps.iam-create.outputs.rgName }}-rg --yes + az group delete --name ${{ steps.iam-create.outputs.rgName }}-rg-identity --yes + notify-failure: name: Notify about failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: e2e-test if: | failure() && @@ -159,15 +186,12 @@ jobs: inputs.scheduled steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Setup bazel uses: ./.github/actions/setup_bazel_nix - with: - useCache: "true" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Notify about failure continue-on-error: true @@ -176,3 +200,4 @@ jobs: projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} test: Windows E2E Test provider: Azure + attestationVariant: "azure-sev-snp" diff --git a/.github/workflows/on-release.yml b/.github/workflows/on-release.yml index 02a94b4a4..73fe2c3b1 100644 --- a/.github/workflows/on-release.yml +++ b/.github/workflows/on-release.yml @@ -15,10 +15,10 @@ on: jobs: complete-release-branch-transaction: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write - contents: read + contents: write env: FULL_VERSION: ${{ github.event.release.tag_name }}${{ github.event.inputs.tag }} outputs: @@ -26,17 +26,13 @@ jobs: WORKING_BRANCH: ${{ env.WORKING_BRANCH }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 # fetch all history - name: Determine branch names run: | - WITHOUT_V=${FULL_VERSION#v} - PART_MAJOR=${WITHOUT_V%%.*} - PART_MINOR=${WITHOUT_V#*.} - MAJOR_MINOR=${PART_MAJOR}.${PART_MINOR} - RELEASE_BRANCH="release/v${MAJOR_MINOR}" + RELEASE_BRANCH="release/${FULL_VERSION%.*}" WORKING_BRANCH="tmp/${FULL_VERSION}" echo "RELEASE_BRANCH=${RELEASE_BRANCH}" | tee -a "$GITHUB_ENV" echo "WORKING_BRANCH=${WORKING_BRANCH}" | tee -a "$GITHUB_ENV" @@ -48,12 +44,12 @@ jobs: git push origin "${WORKING_BRANCH}":"${RELEASE_BRANCH}" update: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: latest: ${{ steps.input-passthrough.outputs.latest }}${{ steps.check-last-release.outputs.latest }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Override latest if: github.event.inputs.latest == 'true' @@ -121,13 +117,62 @@ jobs: add-image-version-to-versionsapi, add-cli-version-to-versionsapi, ] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + permissions: + id-token: write + contents: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Remove temporary branch + run: git push origin --delete "${{needs.complete-release-branch-transaction.outputs.WORKING_BRANCH}}" + + mirror-gcp-mpi: + name: "Mirror GCP Marketplace Image" + needs: [add-image-version-to-versionsapi] + runs-on: ubuntu-24.04 permissions: id-token: write contents: read steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Remove temporary branch - run: git push origin --delete "${WORKING_BRANCH}" + - uses: ./.github/actions/setup_bazel_nix + + - name: Login to AWS + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + with: + role-to-assume: arn:aws:iam::795746500882:role/GitHubConstellationImagePipeline + aws-region: eu-central-1 + + - name: Fetch latest release version + id: fetch-version + uses: ./.github/actions/versionsapi + with: + command: latest + stream: stable + ref: "-" + + - name: Fetch GCP image reference + id: fetch-reference + shell: bash + run: | + aws s3 cp s3://cdn-constellation-backend/constellation/v2/ref/-/stream/stable/${{ steps.fetch-version.outputs.output }}/image/info.json . + FULL_REF=$(yq e -r -oy '.list.[] | select(.attestationVariant == "gcp-sev-snp") | .reference' info.json) + IMAGE_NAME=$(echo "${FULL_REF}" | cut -d / -f 5) + echo "reference=$IMAGE_NAME" | tee -a "$GITHUB_OUTPUT" + + - name: Login to GCP + uses: ./.github/actions/login_gcp + with: + service_account: "mp-image-uploader@edgeless-systems-public.iam.gserviceaccount.com" + + - name: Mirror + shell: bash + run: | + gcloud --project=edgeless-systems-public compute images create ${{ steps.fetch-reference.outputs.reference }} \ + --source-image=${{ steps.fetch-reference.outputs.reference }} \ + --source-image-project=constellation-images \ + --licenses=projects/edgeless-systems-public/global/licenses/cloud-marketplace-c3d24830a0502e29-df1ebeb69c0ba664 diff --git a/.github/workflows/purge-main.yml b/.github/workflows/purge-main.yml index 9bbb40c6a..5a64705c4 100644 --- a/.github/workflows/purge-main.yml +++ b/.github/workflows/purge-main.yml @@ -10,7 +10,7 @@ on: jobs: find-version: name: Delete version from main ref - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: version: ${{ steps.find.outputs.version }} permissions: @@ -18,12 +18,12 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead aws-region: eu-central-1 @@ -47,6 +47,8 @@ jobs: ;; esac + - uses: ./.github/actions/setup_bazel_nix + - name: List versions id: list uses: ./.github/actions/versionsapi diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml new file mode 100644 index 000000000..2699b0895 --- /dev/null +++ b/.github/workflows/release-publish.yml @@ -0,0 +1,79 @@ +name: 'Release: on-publish' + +on: + release: + types: + - published + workflow_dispatch: + inputs: + tag: + description: 'Semantic version tag of the release (vX.Y.Z).' + required: true + +jobs: + post-release-actions: + runs-on: ubuntu-24.04 + permissions: + issues: write + env: + FULL_VERSION: ${{ github.event.release.tag_name }}${{ github.event.inputs.tag }} + GH_TOKEN: ${{ github.token }} + steps: + - name: Mark milestone as complete + run: | + milestones=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/edgelesssys/constellation/milestones) + + current_milestone=$(echo "${milestones}" | jq -r ".[] | select(.title == \"${FULL_VERSION}\")") + echo "current milestone: ${current_milestone}" + if [[ -z "${current_milestone}" ]]; then + echo "milestone ${FULL_VERSION} does not exist, nothing to do..." + exit 0 + fi + + current_milestone_state=$(echo "${current_milestone}" | jq -r '.state') + echo "current milestone state: ${current_milestone_state}" + if [[ "${current_milestone_state}" != "open" ]]; then + echo "milestone ${FULL_VERSION} is already closed, nothing to do..." + exit 0 + fi + + milestone_number=$(echo "${current_milestone}" | jq -r '.number') + echo "milestone number: ${milestone_number}" + if [[ -z "${milestone_number}" ]]; then + echo "failed parsing milestone number" + exit 1 + fi + + gh api \ + --method PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/edgelesssys/constellation/milestones/${milestone_number}" \ + -f state=closed + + - name: Create next milestone + run: | + WITHOUT_V=${FULL_VERSION#v} + PART_MAJOR=${WITHOUT_V%%.*} + PART_MINOR=${WITHOUT_V#*.} + PART_MINOR=${PART_MINOR%%.*} + NEXT_MINOR=v${PART_MAJOR}.$((PART_MINOR + 1)).0 + + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/edgelesssys/constellation/milestones | + jq -r '.[].title' | \ + grep -xqF "${NEXT_MINOR}" && exit 0 + + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/edgelesssys/constellation/milestones \ + -f title="${NEXT_MINOR}" \ + -f state='open' \ + -f "due_on=$(date -d '2 months' +'%Y-%m-%dT00:00:00Z')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bacf65af3..08e09da18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ concurrency: jobs: verify-inputs: name: Verify inputs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: FULL_VERSION: ${{ inputs.version }} outputs: @@ -33,7 +33,7 @@ jobs: RELEASE_BRANCH: ${{ steps.version-info.outputs.RELEASE_BRANCH }} WORKING_BRANCH: ${{ steps.version-info.outputs.WORKING_BRANCH }} steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Working branch run: echo "WORKING_BRANCH=$(git branch --show-current)" | tee -a "$GITHUB_ENV" @@ -72,10 +72,9 @@ jobs: echo "WORKING_BRANCH=${WORKING_BRANCH}" } | tee -a "$GITHUB_OUTPUT" - docs: - name: Create docs release (from main) - runs-on: ubuntu-22.04 - if: inputs.kind == 'minor' + update-main-branch: + name: Update main branch with release changes + runs-on: ubuntu-24.04 needs: verify-inputs permissions: contents: write @@ -85,36 +84,61 @@ jobs: MAJOR_MINOR: ${{ needs.verify-inputs.outputs.MAJOR_MINOR }} BRANCH: docs/${{ needs.verify-inputs.outputs.MAJOR_MINOR }} steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: main + - name: Configure git + run: | + git config --global user.name "edgelessci" + git config --global user.email "edgelessci@users.noreply.github.com" + - name: Create docs release + if: inputs.kind == 'minor' working-directory: docs run: | - npm install + npm ci npm run docusaurus docs:version "${MAJOR_MINOR}" + git add . + git commit -am "docs: release ${MAJOR_MINOR}" + # Clean up auxiliary files, so next steps run on a clean tree + git clean -fdx :/ + + - name: Update version.txt + if: inputs.kind == 'minor' + run: | + pre_release_version="v${{ needs.verify-inputs.outputs.PART_MAJOR }}.$((${{ needs.verify-inputs.outputs.PART_MINOR }} + 1)).0-pre" + echo "${pre_release_version}" > version.txt + git add version.txt + git commit -m "chore: update version.txt to ${pre_release_version}" + + - name: Update CI for new version + run: | + sed -i 's/fromVersion: \["[^"]*"\]/fromVersion: ["${{ inputs.version }}"]/g' .github/workflows/e2e-test-release.yml + sed -i 's/fromVersion: \["[^"]*"\]/fromVersion: ["${{ inputs.version }}"]/g' .github/workflows/e2e-test-weekly.yml - name: Create docs pull request - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: branch: ${{ env.BRANCH }} base: main - title: "docs: add release ${{ env.VERSION }}" + title: "Post ${{ env.VERSION }} release updates to main" body: | :robot: *This is an automated PR.* :robot: The PR is triggered as part of the automated release process of version ${{ env.VERSION }}. - It releases a new version of the documentation. - commit-message: "docs: add release ${{ env.VERSION }}" + commit-message: "chore: update CI for ${{ env.VERSION }}" committer: edgelessci + author: edgelessci labels: no changelog + assignees: ${{ github.actor }} + reviewers: ${{ github.actor }} # We need to push changes using a token, otherwise triggers like on:push and on:pull_request won't work. token: ${{ !github.event.pull_request.head.repo.fork && secrets.CI_COMMIT_PUSH_PR || '' }} check-working-branch: name: Check temporary working branch - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: verify-inputs permissions: contents: write @@ -123,7 +147,7 @@ jobs: WORKING_BRANCH: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} @@ -152,7 +176,7 @@ jobs: update-versions: name: Update container image versions needs: [verify-inputs, check-working-branch] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: contents: write packages: read @@ -161,7 +185,7 @@ jobs: WITHOUT_V: ${{ needs.verify-inputs.outputs.WITHOUT_V }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} @@ -181,7 +205,7 @@ jobs: yq eval -i ".version = \"$WITHOUT_V\"" s3proxy/deploy/s3proxy/Chart.yaml yq eval -i ".image = \"ghcr.io/edgelesssys/constellation/s3proxy:$VERSION\"" s3proxy/deploy/s3proxy/values.yaml - git add s3proxy/deploy/s3proxy/Chart.yaml + git add s3proxy/deploy/s3proxy/Chart.yaml s3proxy/deploy/s3proxy/values.yaml - name: Commit run: | @@ -215,25 +239,41 @@ jobs: stream: "stable" ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} + check-measurements-reproducibility: + name: Check measurements reproducibility + needs: [verify-inputs, os-image] + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} + + - name: Check reproducibility + uses: ./.github/actions/check_measurements_reproducibility + with: + version: ${{ inputs.version }} + ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} + update-hardcoded-measurements: name: Update hardcoded measurements (in the CLI) needs: [verify-inputs, os-image] permissions: contents: write - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: VERSION: ${{ inputs.version }} WITHOUT_V: ${{ needs.verify-inputs.outputs.WITHOUT_V }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} - name: Setup Go environment - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: "1.21.5" + go-version: "1.24.3" cache: true - name: Build generateMeasurements tool @@ -250,8 +290,12 @@ jobs: run: | git config --global user.name "edgelessci" git config --global user.email "edgelessci@users.noreply.github.com" - git commit -m "attestation: hardcode measurements for ${VERSION}" - git push + if git diff-index --quiet HEAD --; then + echo "No changes to commit" + else + git commit -m "attestation: hardcode measurements for ${VERSION}" + git push + fi draft-release: name: Draft release (CLI) @@ -278,6 +322,7 @@ jobs: packages: write id-token: write contents: read + actions: write secrets: inherit with: ref: ${{ needs.verify-inputs.outputs.WORKING_BRANCH }} diff --git a/.github/workflows/reproducible-builds.yml b/.github/workflows/reproducible-builds.yml index 8fd53ba69..41aca0ac3 100644 --- a/.github/workflows/reproducible-builds.yml +++ b/.github/workflows/reproducible-builds.yml @@ -1,10 +1,21 @@ -# Build Constellation CLI and check for reproducible builds +# Verify that Constellation builds are reproducible. +# +# The build-* jobs' matrix has three dimensions: a list of targets to build, a +# list of runners to build on and a method of installing dependencies. The +# produced binaries and OS images are expected to be bit-for-bit identical, +# without any dependencies on the runtime setup details. +# +# The compare-* jobs only have the target dimension. They obtain the built +# targets from all runners and check that there are no diffs between them. name: Reproducible Builds on: workflow_dispatch: schedule: - cron: "45 06 * * 1" # Every Monday at 6:45am + pull_request: + paths: + - .github/workflows/reproducible-builds.yml jobs: build-binaries: @@ -17,22 +28,39 @@ jobs: - "cli_enterprise_linux_amd64" - "cli_enterprise_linux_arm64" - "cli_enterprise_windows_amd64" - runner: ["ubuntu-22.04", "ubuntu-20.04"] + runner: + - "ubuntu-24.04" + - "ubuntu-22.04" + deps: + - conventional + - eccentric env: bazel_target: "//cli:${{ matrix.target }}" - binary: "${{ matrix.target }}-${{ matrix.runner }}" + binary: "${{ matrix.target }}-${{ matrix.runner }}-${{ matrix.deps }}" runs-on: ${{ matrix.runner }} steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - - name: Setup bazel + - name: Setup dependencies uses: ./.github/actions/setup_bazel_nix - with: - useCache: "logs" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + if: matrix.deps == 'conventional' + + - name: Setup dependencies (eccentric) + if: matrix.deps == 'eccentric' + run: | + bazelVersion=$(cat .bazelversion) + mkdir -p "$HOME/.local/bin" + curl -fsSL -o "$HOME/.local/bin/bazel" "https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-linux-x86_64" + chmod a+x "$HOME/.local/bin/bazel" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + curl -fsSL -o "$HOME/.local/bin/nix-installer" https://github.com/DeterminateSystems/nix-installer/releases/download/v3.4.2/nix-installer-x86_64-linux # renovate:github-release + nixVersion=$(cat .nixversion) + chmod a+x "$HOME/.local/bin/nix-installer" + "$HOME/.local/bin/nix-installer" install --no-confirm --nix-package-url "https://releases.nixos.org/nix/nix-$nixVersion/nix-$nixVersion-x86_64-linux.tar.xz" - name: Build shell: bash @@ -53,15 +81,15 @@ jobs: run: shasum -a 256 "${binary}" | tee "${binary}.sha256" - name: Upload binary artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: "binaries-${{ matrix.target }}" + name: "binaries-${{ matrix.target }}-${{ matrix.runner }}-${{ matrix.deps }}" path: "${{ env.binary }}" - name: Upload hash artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: "sha256sums" + name: "sha256sums-${{ matrix.target }}-${{ matrix.runner }}-${{ matrix.deps }}" path: "${{ env.binary }}.sha256" build-osimages: @@ -73,22 +101,31 @@ jobs: - "aws_aws-nitro-tpm_console" - "qemu_qemu-vtpm_debug" - "gcp_gcp-sev-snp_nightly" - runner: ["ubuntu-22.04", "ubuntu-20.04"] + runner: ["ubuntu-24.04", "ubuntu-22.04"] env: bazel_target: "//image/system:${{ matrix.target }}" binary: "osimage-${{ matrix.target }}-${{ matrix.runner }}" runs-on: ${{ matrix.runner }} steps: + - name: Remove security hardening features + if: matrix.runner == 'ubuntu-24.04' + shell: bash + run: | + # Taken from https://github.com/systemd/mkosi/blob/fcacc94b9f72d9b6b1f03779b0c6e07209ceb54b/action.yaml#L42-L57. + sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_unconfined=0 + sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_userns=0 + # This command fails with a non-zero error code even though it unloads the apparmor profiles. + # https://gitlab.com/apparmor/apparmor/-/issues/403 + sudo aa-teardown || true + sudo apt-get remove -y apparmor + - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Setup bazel uses: ./.github/actions/setup_bazel_nix - with: - useCache: "logs" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Build shell: bash @@ -109,15 +146,15 @@ jobs: run: shasum -a 256 "${binary}" | tee "${binary}.sha256" - name: Upload binary artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: "osimages-${{ matrix.target }}" + name: "osimages-${{ matrix.target }}-${{ matrix.runner }}" path: "${{ env.binary }}" - name: Upload hash artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: "sha256sums" + name: "sha256sums-${{ matrix.target }}-${{ matrix.runner }}" path: "${{ env.binary }}.sha256" compare-binaries: @@ -131,24 +168,29 @@ jobs: - "cli_enterprise_linux_amd64" - "cli_enterprise_linux_arm64" - "cli_enterprise_windows_amd64" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - name: Download binaries - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: "binaries-${{ matrix.target }}" + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - - name: Hash - shell: bash - if: runner.os == 'Linux' - run: sha256sum cli_enterprise* + - name: Download binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: "binaries-${{ matrix.target }}-*" + merge-multiple: true - - name: Compare binaries - shell: bash - run: | - # shellcheck disable=SC2207,SC2116 - list=($(echo "cli_enterprise*")) - diff -s --to-file="${list[0]}" "${list[@]:1}" | tee "${GITHUB_STEP_SUMMARY}" + - name: Hash + shell: bash + if: runner.os == 'Linux' + run: sha256sum cli_enterprise* + + - name: Compare binaries + shell: bash + run: | + # shellcheck disable=SC2207,SC2116 + list=($(echo "cli_enterprise*")) + diff -s --to-file="${list[0]}" "${list[@]:1}" | tee "${GITHUB_STEP_SUMMARY}" compare-osimages: needs: build-osimages @@ -160,21 +202,26 @@ jobs: - "aws_aws-nitro-tpm_console" - "qemu_qemu-vtpm_debug" - "gcp_gcp-sev-snp_nightly" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - name: Download os images - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: "osimages-${{ matrix.target }}" + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - - name: Hash - shell: bash - if: runner.os == 'Linux' - run: sha256sum osimage-* + - name: Download os images + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: "osimages-${{ matrix.target }}-*" + merge-multiple: true - - name: Compare os images - shell: bash - run: | - # shellcheck disable=SC2207,SC2116 - list=($(echo "osimage-*")) - diff -s --to-file="${list[0]}" "${list[@]:1}" | tee "${GITHUB_STEP_SUMMARY}" + - name: Hash + shell: bash + if: runner.os == 'Linux' + run: sha256sum osimage-* + + - name: Compare os images + shell: bash + run: | + # shellcheck disable=SC2207,SC2116 + list=($(echo "osimage-*")) + diff -s --to-file="${list[0]}" "${list[@]:1}" | tee "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 90efc76fc..08a7faeac 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -9,7 +9,7 @@ on: jobs: analysis: name: Scorecard analysis - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: # Needed to upload the results to code-scanning dashboard. security-events: write @@ -18,25 +18,25 @@ jobs: steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif publish_results: true - name: Upload artifact - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: SARIF file path: results.sarif retention-days: 5 - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1 + uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 with: sarif_file: results.sarif diff --git a/.github/workflows/sync-terraform-docs.yml b/.github/workflows/sync-terraform-docs.yml index 184df8283..9bc2aac07 100644 --- a/.github/workflows/sync-terraform-docs.yml +++ b/.github/workflows/sync-terraform-docs.yml @@ -18,14 +18,14 @@ jobs: pull-requests: write steps: - name: Checkout constellation repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} fetch-depth: 0 path: constellation - name: Checkout terraform-provider-constellation repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: edgelesssys/terraform-provider-constellation ref: main @@ -40,7 +40,7 @@ jobs: - name: Create pull request id: create-pull-request - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: path: terraform-provider-constellation branch: "feat/docs/update" @@ -58,7 +58,7 @@ jobs: delete-branch: true - name: Merge pull request - uses: peter-evans/enable-pull-request-automerge@v3 + uses: peter-evans/enable-pull-request-automerge@a660677d5469627102a1c1e11409dd063606628d # v3.0.0 with: pull-request-number: ${{ steps.create-pull-request.outputs.pull-request-number }} merge-method: squash diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index bb65363a2..c6908ff3d 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -20,22 +20,19 @@ on: jobs: integration-test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: CTEST_OUTPUT_ON_FAILURE: True steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Setup bazel uses: ./.github/actions/setup_bazel_nix - with: - useCache: "true" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Integration Tests env: TMPDIR: ${{ runner.temp }} - run: sudo -E "PATH=$PATH" bazel test //... --config=nostamp --remote_download_minimal --config=integration --spawn_strategy=standalone + run: sudo -E "PATH=$PATH" bazel test //... --config=nostamp --remote_download_minimal --config=integration-only --spawn_strategy=standalone diff --git a/.github/workflows/test-operator-codegen.yml b/.github/workflows/test-operator-codegen.yml index 5f693c56f..b2f92ba7f 100644 --- a/.github/workflows/test-operator-codegen.yml +++ b/.github/workflows/test-operator-codegen.yml @@ -18,17 +18,17 @@ on: jobs: govulncheck: name: check-codegen - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Setup Go environment - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: - go-version: "1.21.5" + go-version: "1.24.3" cache: true - name: Run code generation diff --git a/.github/workflows/test-tfsec.yml b/.github/workflows/test-tfsec.yml index 72fb9c3c4..5517ac887 100644 --- a/.github/workflows/test-tfsec.yml +++ b/.github/workflows/test-tfsec.yml @@ -17,13 +17,13 @@ on: jobs: tfsec: name: tfsec - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: contents: read pull-requests: write steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} diff --git a/.github/workflows/test-tidy.yml b/.github/workflows/test-tidy.yml index 6adb624d6..25f06e174 100644 --- a/.github/workflows/test-tidy.yml +++ b/.github/workflows/test-tidy.yml @@ -17,7 +17,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} # No token available for forks, so we can't push changes @@ -34,11 +34,10 @@ jobs: with: useCache: "rbe" rbePlatform: "ubuntu-22.04" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Assume AWS role to upload Bazel dependencies to S3 if: startsWith(github.head_ref, 'renovate/') - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationMirrorWrite aws-region: eu-central-1 @@ -52,7 +51,9 @@ jobs: - name: Run Bazel tidy shell: bash - run: bazel run //:tidy + run: | + bazel run //:tidy + bazel mod deps --lockfile_mode=update - name: Check if untidy id: untidy @@ -97,10 +98,11 @@ jobs: exit 0 fi + # Use quadruple backticks to escape triple backticks in diff'ed files. cat << EOF >> "${GITHUB_STEP_SUMMARY}" - \`\`\`diff + \`\`\`\`diff ${diff} - \`\`\` + \`\`\`\` EOF if [[ "${{ steps.untidy.outputs.untidy }}" == "true" ]] && diff --git a/.github/workflows/test-unittest.yml b/.github/workflows/test-unittest.yml index 18b8f8915..7b64254dd 100644 --- a/.github/workflows/test-unittest.yml +++ b/.github/workflows/test-unittest.yml @@ -30,7 +30,7 @@ jobs: pull-requests: write steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} fetch-depth: 0 @@ -40,7 +40,6 @@ jobs: with: useCache: "rbe" rbePlatform: "ubuntu-22.04" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Install AWS cli run: | @@ -50,7 +49,7 @@ jobs: rm -rf awscliv2.zip aws - name: Login to AWS (IAM role) - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionGocoverage aws-region: eu-central-1 @@ -70,7 +69,7 @@ jobs: - name: Comment coverage if: steps.coverage.outputs.uploadable == 'true' && github.event_name == 'pull_request' - uses: marocchino/sticky-pull-request-comment@efaaab3fd41a9c3de579aba759d2552635e590fd # v2.8.0 + uses: marocchino/sticky-pull-request-comment@67d0dec7b07ed060a405f9b2a64b8ab319fdd7db # v2.9.2 with: header: coverage path: coverage_diff.md diff --git a/.github/workflows/update-rpms.yml b/.github/workflows/update-rpms.yml index 0859540cc..c7146607e 100644 --- a/.github/workflows/update-rpms.yml +++ b/.github/workflows/update-rpms.yml @@ -7,25 +7,24 @@ on: jobs: update-rpms: - runs-on: "ubuntu-22.04" + runs-on: "ubuntu-24.04" permissions: id-token: write contents: read steps: - name: Checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + token: ${{ secrets.CI_COMMIT_PUSH_PR }} - name: Assume AWS role to upload Bazel dependencies to S3 - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationMirrorWrite aws-region: eu-central-1 - name: Setup bazel uses: ./.github/actions/setup_bazel_nix - with: - useCache: "true" - buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - name: Update rpms run: bazel run //image/mirror:update_packages @@ -40,7 +39,7 @@ jobs: fi - name: Create pull request - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 # v5.0.2 + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 with: branch: "image/automated/update-rpms-${{ github.run_number }}" base: main @@ -52,6 +51,7 @@ jobs: It updates the locked rpm packages that form the Constellation OS images. commit-message: "image: update locked rpms" committer: edgelessci + author: edgelessci labels: dependencies # We need to push changes using a token, otherwise triggers like on:push and on:pull_request won't work. token: ${{ !github.event.pull_request.head.repo.fork && secrets.CI_COMMIT_PUSH_PR || '' }} diff --git a/.github/workflows/versionsapi.yml b/.github/workflows/versionsapi.yml index f4bc42117..27acd9287 100644 --- a/.github/workflows/versionsapi.yml +++ b/.github/workflows/versionsapi.yml @@ -106,7 +106,7 @@ concurrency: jobs: versionsapi: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: id-token: write contents: read @@ -115,7 +115,7 @@ jobs: steps: - name: Check out repository id: checkout - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} @@ -149,21 +149,21 @@ jobs: - name: Login to AWS without write access if: steps.check-rights.outputs.write == 'false' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead aws-region: eu-central-1 - name: Login to AWS with write access if: steps.check-rights.outputs.write == 'true' && steps.check-rights.outputs.auth == 'false' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIWrite aws-region: eu-central-1 - name: Login to AWS with write and image remove access if: steps.check-rights.outputs.write == 'true' && steps.check-rights.outputs.auth == 'true' - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRemove aws-region: eu-central-1 @@ -180,6 +180,8 @@ jobs: with: service_account: "image-deleter@constellation-images.iam.gserviceaccount.com" + - uses: ./.github/actions/setup_bazel_nix + - name: Execute versionsapi CLI id: run uses: ./.github/actions/versionsapi diff --git a/.gitignore b/.gitignore index 3d97460a6..27a599676 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,9 @@ __pycache__/ # s3proxy misc files port-forward.log s3proxy-ca.crt + +# Lychee link checker +.lycheecache + +# direnv +.direnv \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 2998b1416..2e5cd2d5c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,51 +1,65 @@ +version: "2" run: - timeout: 10m build-tags: - integration - e2e modules-download-mode: readonly - skip-dirs: - - 3rdparty/node-maintenance-operator - output: - format: tab - sort-results: true - + formats: + tab: + path: stdout + colors: false linters: enable: - # Default linters - - errcheck - - gosimple - - govet - - ineffassign - - staticcheck - - typecheck - - unused - # Additional linters - bodyclose + - copyloopvar - errname - - exportloopref - godot - - gofmt - - gofumpt - misspell - noctx - revive - - tenv - unconvert - unparam + - usetesting + settings: + errcheck: + exclude-functions: + - (*go.uber.org/zap.Logger).Sync + - (*google.golang.org/grpc.Server).Serve + exclusions: + generated: lax + presets: + - common-false-positives + - legacy + - std-error-handling + paths: + - 3rdparty/node-maintenance-operator + rules: + # TODO(burgerdev): these exclusions have been added to ease migration to v2 and should eventually be addressed. + - linters: ["staticcheck"] + text: "QF1008: could remove embedded field" + - linters: ["staticcheck"] + text: "QF1001: could apply De Morgan's law" + - linters: ["staticcheck"] + text: "ST1005: error strings should not be capitalized" + - linters: ["revive"] + text: "package-comments: package comment should be of the form" + - linters: ["revive"] + text: "package-comments: should have a package comment" + - linters: ["staticcheck"] + text: "QF1012: Use fmt.Fprintf" + - linters: ["staticcheck"] + text: "ST1019" + issues: max-issues-per-linter: 0 max-same-issues: 20 - include: - - EXC0012 - - EXC0014 - -linters-settings: - errcheck: - # List of functions to exclude from checking, where each entry is a single function to exclude. - # See https://github.com/kisielk/errcheck#excluding-functions for details. - exclude-functions: - - (*go.uber.org/zap.Logger).Sync - - (*google.golang.org/grpc.Server).Serve +formatters: + enable: + - gofmt + - gofumpt + exclusions: + generated: lax + paths: + - 3rdparty/node-maintenance-operator diff --git a/.lychee.toml b/.lychee.toml new file mode 100644 index 000000000..70b1a2652 --- /dev/null +++ b/.lychee.toml @@ -0,0 +1,51 @@ +# Verbose program output +# Accepts log level: "error", "warn", "info", "debug", "trace" +verbose = "info" + +# Don't show interactive progress bar while checking links. +no_progress = true + +# Enable link caching. This can be helpful to avoid checking the same links on +# multiple runs. +cache = true + +# Discard all cached requests older than this duration. +max_cache_age = "10m" + +# Maximum number of concurrent link checks. +max_concurrency = 5 + +# Comma-separated list of accepted status codes for valid links. +# accept = ["200"] + +# Request method +method = "get" + +# Check links inside `` and `
` blocks as well as Markdown code
+# blocks.
+include_verbatim = false
+
+# Exclude URLs and mail addresses from checking (supports regex).
+exclude = [
+    # Blocked for robots.
+    '^https://twitter\.com',
+    # Only available when logged in.
+    '^https://portal\.azure\.com/',
+    # The Wireguard website sproadically returns 500.
+    '^https://www\.wireguard\.com/',
+    # venturebeat detects our link checker
+    '^https://venturebeat\.com/',
+    # dev-docs reference the internal wiki
+    '^https://github\.com/edgelesssys/wiki',
+]
+
+# Exclude these filesystem paths from getting checked.
+exclude_path = ["internal/constellation/helm/charts/cilium"]
+
+# Exclude all private IPs from checking.
+# Equivalent to setting `exclude_private`, `exclude_link_local`, and
+# `exclude_loopback` to true.
+exclude_all_private = true
+
+# Check mail addresses
+include_mail = false
diff --git a/.lycheeignore b/.lycheeignore
deleted file mode 100644
index 128bc8384..000000000
--- a/.lycheeignore
+++ /dev/null
@@ -1,7 +0,0 @@
-http://localhost:
-# TODO: Remove when site fixed their TLS chain ()
-https://www.ntia.gov/SBOM
-# Twitter times out on request, maybe someone pulled the plug of this subsystem
-https://twitter.com/EdgelessSystems
-# Azure portal internal link, will return 403 for crawlers, or error when not logged in
-https://portal.azure.com/#view/HubsExtension/BrowseResource/resourceType/Microsoft.Compute%2Fgalleries
diff --git a/.nixversion b/.nixversion
new file mode 100644
index 000000000..40a8d7f12
--- /dev/null
+++ b/.nixversion
@@ -0,0 +1 @@
+2.25.2
diff --git a/.vale.ini b/.vale.ini
index 36a01c5ef..4b481e121 100644
--- a/.vale.ini
+++ b/.vale.ini
@@ -1,13 +1,9 @@
 StylesPath = docs/styles
-Vocab = constellation
-
-# IgnoredScopes specifies inline-level HTML tags to ignore.
-# These tags may occur in an active scope (unlike SkippedScopes, skipped entirely) but their content still will not raise any alerts.
-# Default: ignore `code` and `tt`.
-IgnoredScopes = code, tt, img
+Vocab = edgeless
 
 [*.md]
 BasedOnStyles = Vale, Microsoft, Google
+Vale.Terms = NO
 
 # decrease to suggestion
 Microsoft.Foreign = suggestion  # conflicts with Microsoft.Contractions
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.need b/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.need
index d3cf0338e..8542c18a4 100644
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.need
+++ b/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.need
@@ -8,10 +8,7 @@ go_library(
         "internal_cross.go",
     ],
     cgo = True,
-    cdeps = select({
-        "@//bazel/settings:tpm_simulator_enabled": ["//simulator/ms-tpm-20-ref:ms_tpm_20_ref"],
-        "//conditions:default": ["@//3rdparty/bazel/com_github_google_go_tpm_tools/placeholder:ms_tpm_20_ref_disabled"],
-    }),
+    cdeps = ["//simulator/ms-tpm-20-ref:ms_tpm_20_ref"],
     copts = [
         "-fno-sanitize=all",  # relax sanitizer checks for this test-only dependency
     ],
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.orig b/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.orig
index 3fc9a7d32..bb03b408a 100644
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.orig
+++ b/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.orig
@@ -73,9 +73,15 @@ go_library(
         "@io_bazel_rules_go//go/platform:openbsd": [
             "-fstack-protector-all",
         ],
+        "@io_bazel_rules_go//go/platform:osx": [
+            "-fstack-protector-all",
+        ],
         "@io_bazel_rules_go//go/platform:plan9": [
             "-fstack-protector-all",
         ],
+        "@io_bazel_rules_go//go/platform:qnx": [
+            "-fstack-protector-all",
+        ],
         "@io_bazel_rules_go//go/platform:solaris": [
             "-fstack-protector-all",
         ],
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.patch b/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.patch
index 238e50f9e..ad04c2292 100644
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.patch
+++ b/3rdparty/bazel/com_github_google_go_tpm_tools/com_github_google_go_tpm_tools.patch
@@ -1,6 +1,6 @@
 --- simulator/internal/BUILD.bazel
 +++ simulator/internal/BUILD.bazel
-@@ -4,83 +4,17 @@
+@@ -4,89 +4,14 @@ go_library(
      name = "internal",
      srcs = [
          "doc.go",
@@ -17,10 +17,8 @@
 -            "-L/usr/local/opt/openssl/lib",
 -        ],
 -        "//conditions:default": [],
-+    cdeps =  select({
-+        "@//bazel/settings:tpm_simulator_enabled": ["//simulator/ms-tpm-20-ref:ms_tpm_20_ref"],
-+        "//conditions:default": ["@//3rdparty/bazel/com_github_google_go_tpm_tools/placeholder:ms_tpm_20_ref_disabled"],
-     }),
+-    }),
++    cdeps = ["//simulator/ms-tpm-20-ref:ms_tpm_20_ref"],
      copts = [
 -        "-DALG_SHA512=ALG_YES",
 -        "-DCERTIFYX509_DEBUG=NO",
@@ -76,9 +74,15 @@
 -        "@io_bazel_rules_go//go/platform:openbsd": [
 -            "-fstack-protector-all",
 -        ],
+-        "@io_bazel_rules_go//go/platform:osx": [
+-            "-fstack-protector-all",
+-        ],
 -        "@io_bazel_rules_go//go/platform:plan9": [
 -            "-fstack-protector-all",
 -        ],
+-        "@io_bazel_rules_go//go/platform:qnx": [
+-            "-fstack-protector-all",
+-        ],
 -        "@io_bazel_rules_go//go/platform:solaris": [
 -            "-fstack-protector-all",
 -        ],
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.need b/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.need
index 83b388537..7758874e6 100644
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.need
+++ b/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.need
@@ -229,8 +229,10 @@ cc_library(
     ],
     deps = [
         ":ms_tpm_20_ref_google_samples",
-        "@//nix/cc:org_openssl",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux_arm64": ["@@org_openssl_aarch64-linux//:org_openssl"],
+        "@io_bazel_rules_go//go/platform:linux_amd64": ["@@org_openssl_x86_64-linux//:org_openssl"],
+    }),
     target_compatible_with = [
         "@platforms//os:linux",
     ]
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.patch b/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.patch
index e98141b8a..8340c947f 100644
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.patch
+++ b/3rdparty/bazel/com_github_google_go_tpm_tools/ms_tpm_20_ref.patch
@@ -1,6 +1,6 @@
 --- simulator/ms-tpm-20-ref/BUILD.bazel
 +++ simulator/ms-tpm-20-ref/BUILD.bazel
-@@ -0,0 +1,512 @@
+@@ -0,0 +1,516 @@
 +cc_library(
 +    name = "ms_tpm_20_ref",
 +    visibility = ["//visibility:public"],
@@ -229,10 +229,14 @@
 +    ],
 +    deps = [
 +        ":ms_tpm_20_ref_google_samples",
-+        "@//nix/cc:org_openssl",
-+    ],
++    ] + select({
++        "@io_bazel_rules_go//go/platform:darwin_arm64": ["@@org_openssl_aarch64-darwin//:org_openssl"],
++        "@io_bazel_rules_go//go/platform:darwin_amd64": ["@@org_openssl_x86_64-darwin//:org_openssl"],
++        "@io_bazel_rules_go//go/platform:linux_arm64": ["@@org_openssl_aarch64-linux//:org_openssl"],
++        "@io_bazel_rules_go//go/platform:linux_amd64": ["@@org_openssl_x86_64-linux//:org_openssl"],
++    }),
 +    target_compatible_with = [
-+        "@platforms//os:linux",
++        "@@platforms//os:linux",
 +    ],
 +)
 +
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/BUILD.bazel b/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/BUILD.bazel
deleted file mode 100644
index 07d713e46..000000000
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/BUILD.bazel
+++ /dev/null
@@ -1,12 +0,0 @@
-load("@rules_cc//cc:defs.bzl", "cc_library")
-
-cc_library(
-    name = "ms_tpm_20_ref_disabled",
-    srcs = ["ms_tpm_20_disabled.c"],
-    hdrs = [
-        "Platform.h",
-        "Tpm.h",
-    ],
-    includes = ["."],
-    visibility = ["//visibility:public"],
-)
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/Platform.h b/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/Platform.h
deleted file mode 100644
index 8aaa55d10..000000000
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/Platform.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#include 
-#include 
-#include 
-
-extern int g_inFailureMode;
-
-typedef union {
-	uint16_t      size;
-	uint8_t       *buffer;
-} TPM2B, TPM2B_SEED;
-typedef struct
-{
-    TPM2B_SEED          EPSeed;
-    TPM2B_SEED          SPSeed;
-    TPM2B_SEED          PPSeed;
-} PERSISTENT_DATA;
-
-extern PERSISTENT_DATA  gp;
-
-void _plat__Reset(bool forceManufacture);
-void _plat__RunCommand(uint32_t requestSize, unsigned char *request,
-                       uint32_t *responseSize, unsigned char **response);
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/Tpm.h b/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/Tpm.h
deleted file mode 100644
index 2e94e1749..000000000
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/Tpm.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#undef TRUE
-#define TRUE                1
-#undef FALSE
-#define FALSE               0
-#undef YES
-#define YES                 1
-#undef NO
-#define NO                  0
-#undef SET
-#define SET                 1
-#undef CLEAR
-#define CLEAR               0
-#ifndef MAX_RESPONSE_SIZE
-#define MAX_RESPONSE_SIZE               4096
-#endif
-
-#ifndef EPSeed
-#define EPSeed 1
-#endif
-#ifndef SPSeed
-#define SPSeed 1
-#endif
-#ifndef PPSeed
-#define PPSeed 1
-#endif
-
-#define NV_SYNC_PERSISTENT(x)
diff --git a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/ms_tpm_20_disabled.c b/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/ms_tpm_20_disabled.c
deleted file mode 100644
index a56d7f43c..000000000
--- a/3rdparty/bazel/com_github_google_go_tpm_tools/placeholder/ms_tpm_20_disabled.c
+++ /dev/null
@@ -1,9 +0,0 @@
-#include "Platform.h"
-#include "Tpm.h"
-
-int g_inFailureMode = 0;
-
-void _plat__Reset(bool forceManufacture) {}
-
-void _plat__RunCommand(uint32_t requestSize, unsigned char *request,
-                       uint32_t *responseSize, unsigned char **response) {}
diff --git a/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/BUILD.bazel b/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/BUILD.bazel
new file mode 100644
index 000000000..9aa7e3f64
--- /dev/null
+++ b/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/BUILD.bazel
@@ -0,0 +1,13 @@
+load("//bazel/sh:def.bzl", "sh_template")
+
+sh_template(
+    name = "pull_files",
+    data = [
+        "@com_github_kubernetes_sigs_aws_load_balancer_controller//:lb_policy",
+    ],
+    substitutions = {
+        "@@POLICY_SRC@@": "$(rootpath @com_github_kubernetes_sigs_aws_load_balancer_controller//:lb_policy)",
+    },
+    template = "pull_files.sh",
+    visibility = ["//visibility:public"],
+)
diff --git a/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/pull_files.sh b/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/pull_files.sh
new file mode 100644
index 000000000..10a224890
--- /dev/null
+++ b/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/pull_files.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+###### script header ######
+
+lib=$(realpath @@BASE_LIB@@) || exit 1
+stat "${lib}" >> /dev/null || exit 1
+
+# shellcheck source=../../../bazel/sh/lib.bash
+if ! source "${lib}"; then
+  echo "Error: could not find import"
+  exit 1
+fi
+
+controller_policy_source="@@POLICY_SRC@@"
+
+###### script body ######
+
+controller_policy_real_source=$(realpath "${controller_policy_source}")
+
+cd "${BUILD_WORKSPACE_DIRECTORY}" # needs to be done after realpath
+
+targetDir="terraform/infrastructure/iam/aws/alb_policy.json"
+
+cp "${controller_policy_real_source}" "${targetDir}"
diff --git a/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/source.bzl b/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/source.bzl
new file mode 100644
index 000000000..e2c1e8034
--- /dev/null
+++ b/3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller/source.bzl
@@ -0,0 +1,22 @@
+"""A module defining the source of the AWS load balancer controller."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def aws_load_balancer_controller_deps():
+    http_archive(
+        name = "com_github_kubernetes_sigs_aws_load_balancer_controller",
+        urls = [
+            "https://cdn.confidential.cloud/constellation/cas/sha256/422af7c03ebc73e1be6aea563475ec9ea6396071fa03158b9a3984aa621b8cb1",
+            "https://github.com/kubernetes-sigs/aws-load-balancer-controller/archive/refs/tags/v2.12.0.tar.gz",
+        ],
+        strip_prefix = "aws-load-balancer-controller-2.12.0",
+        build_file_content = """
+filegroup(
+    srcs = ["docs/install/iam_policy.json"],
+    name = "lb_policy",
+    visibility = ["//visibility:public"],
+)
+        """,
+        type = "tar.gz",
+        sha256 = "422af7c03ebc73e1be6aea563475ec9ea6396071fa03158b9a3984aa621b8cb1",
+    )
diff --git a/3rdparty/bazel/com_github_martinjungblut_go_cryptsetup/com_github_martinjungblut_go_cryptsetup.patch b/3rdparty/bazel/com_github_martinjungblut_go_cryptsetup/com_github_martinjungblut_go_cryptsetup.patch
index f87c1a730..0319cc797 100644
--- a/3rdparty/bazel/com_github_martinjungblut_go_cryptsetup/com_github_martinjungblut_go_cryptsetup.patch
+++ b/3rdparty/bazel/com_github_martinjungblut_go_cryptsetup/com_github_martinjungblut_go_cryptsetup.patch
@@ -4,9 +4,9 @@
          "plain.go",
      ],
      cgo = True,
-+    cdeps = [
-+        "@//nix/cc:cryptsetup",
-+    ],
++    cdeps = select({
++        "@io_bazel_rules_go//go/platform:linux_amd64": ["@@cryptsetup_x86_64-linux//:cryptsetup"],
++    }),
      importpath = "github.com/martinjungblut/go-cryptsetup",
      visibility = ["//visibility:public"],
  )
diff --git a/3rdparty/bazel/com_github_medik8s_node_maintainance_operator/source.bzl b/3rdparty/bazel/com_github_medik8s_node_maintainance_operator/source.bzl
index a9d20a08c..aa81fc947 100644
--- a/3rdparty/bazel/com_github_medik8s_node_maintainance_operator/source.bzl
+++ b/3rdparty/bazel/com_github_medik8s_node_maintainance_operator/source.bzl
@@ -6,10 +6,10 @@ def node_maintainance_operator_deps():
     http_archive(
         name = "com_github_medik8s_node_maintainance_operator",
         urls = [
-            "https://cdn.confidential.cloud/constellation/cas/sha256/df5ea2f9d982dd78770f2549333fd40aaf40e50a28deec9d7892f83cf9d1bdb2",
-            "https://github.com/medik8s/node-maintenance-operator/archive/refs/tags/v0.15.0.tar.gz",
+            "https://cdn.confidential.cloud/constellation/cas/sha256/6ccc7f152e5c595ab24eaadcda77870101eccc482694dc6f0d93be2528406ae2",
+            "https://github.com/medik8s/node-maintenance-operator/archive/refs/tags/v0.17.0.tar.gz",
         ],
-        strip_prefix = "node-maintenance-operator-0.15.0",
+        strip_prefix = "node-maintenance-operator-0.17.0",
         build_file_content = """
 api_v1beta1 = glob(["api/v1beta1/*.go"])
 filegroup(
@@ -19,5 +19,5 @@ filegroup(
 )
         """,
         type = "tar.gz",
-        sha256 = "df5ea2f9d982dd78770f2549333fd40aaf40e50a28deec9d7892f83cf9d1bdb2",
+        sha256 = "6ccc7f152e5c595ab24eaadcda77870101eccc482694dc6f0d93be2528406ae2",
     )
diff --git a/3rdparty/bazel/org_golang/BUILD.bazel b/3rdparty/bazel/org_golang/BUILD.bazel
new file mode 100644
index 000000000..dc940d416
--- /dev/null
+++ b/3rdparty/bazel/org_golang/BUILD.bazel
@@ -0,0 +1 @@
+exports_files(["go_tls_max_handshake_size.patch"])
diff --git a/3rdparty/bazel/org_golang/go_tls_max_handshake_size.patch b/3rdparty/bazel/org_golang/go_tls_max_handshake_size.patch
new file mode 100644
index 000000000..5370f99e7
--- /dev/null
+++ b/3rdparty/bazel/org_golang/go_tls_max_handshake_size.patch
@@ -0,0 +1,11 @@
+--- src/crypto/tls/common.go
++++ src/crypto/tls/common.go
+@@ -64,7 +64,7 @@ const (
+ 	maxCiphertext              = 16384 + 2048 // maximum ciphertext payload length
+ 	maxCiphertextTLS13         = 16384 + 256  // maximum ciphertext length in TLS 1.3
+ 	recordHeaderLen            = 5            // record header length
+-	maxHandshake               = 65536        // maximum handshake we support (protocol max is 16 MB)
++	maxHandshake               = 262144        // maximum handshake we support (protocol max is 16 MB)
+ 	maxHandshakeCertificateMsg = 262144       // maximum certificate message size (256 KiB)
+ 	maxUselessRecords          = 16           // maximum number of consecutive non-advancing records
+ )
diff --git a/3rdparty/bazel/org_libvirt_go_libvirt/go_libvirt.patch b/3rdparty/bazel/org_libvirt_go_libvirt/go_libvirt.patch
index d2c799cae..78150041c 100644
--- a/3rdparty/bazel/org_libvirt_go_libvirt/go_libvirt.patch
+++ b/3rdparty/bazel/org_libvirt_go_libvirt/go_libvirt.patch
@@ -5,7 +5,7 @@
          "typedparams.go",
      ],
 +    cdeps = [
-+        "@//nix/cc:libvirt",
++        "@@libvirt_x86_64-linux//:libvirt",
 +    ],
      cgo = True,
      importpath = "libvirt.org/go/libvirt",
diff --git a/3rdparty/gcp-guest-agent/Dockerfile b/3rdparty/gcp-guest-agent/Dockerfile
index a6ee7a351..d985595db 100644
--- a/3rdparty/gcp-guest-agent/Dockerfile
+++ b/3rdparty/gcp-guest-agent/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f as build
+FROM ubuntu:24.04@sha256:b59d21599a2b151e23eea5f6602f4af4d7d31c4e236d22bf0b62b86d2e386b8f as build
 
 # Install packages
 RUN apt-get update && apt-get install -y \
@@ -6,7 +6,7 @@ RUN apt-get update && apt-get install -y \
     git
 
 # Install Go
-ARG GO_VER=1.21.5
+ARG GO_VER=1.24.4
 RUN wget -q https://go.dev/dl/go${GO_VER}.linux-amd64.tar.gz && \
     tar -C /usr/local -xzf go${GO_VER}.linux-amd64.tar.gz && \
     rm go${GO_VER}.linux-amd64.tar.gz
diff --git a/3rdparty/node-maintenance-operator/go.mod b/3rdparty/node-maintenance-operator/go.mod
deleted file mode 100644
index f1cba4c06..000000000
--- a/3rdparty/node-maintenance-operator/go.mod
+++ /dev/null
@@ -1,25 +0,0 @@
-module github.com/edgelesssys/constellation/v2/3rdparty/node-maintenance-operator
-
-go 1.21
-
-require (
-	k8s.io/apimachinery v0.27.3
-	sigs.k8s.io/controller-runtime v0.15.0
-)
-
-require (
-	github.com/go-logr/logr v1.2.4 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-)
diff --git a/3rdparty/node-maintenance-operator/go.sum b/3rdparty/node-maintenance-operator/go.sum
deleted file mode 100644
index fc814ba04..000000000
--- a/3rdparty/node-maintenance-operator/go.sum
+++ /dev/null
@@ -1,98 +0,0 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
-github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU=
-github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
-golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.27.2 h1:+H17AJpUMvl+clT+BPnKf0E3ksMAzoBBg7CntpSuADo=
-k8s.io/api v0.27.2/go.mod h1:ENmbocXfBT2ADujUXcBhHV55RIT31IIEvkntP6vZKS4=
-k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM=
-k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
-k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=
-k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk=
-k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU=
-sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk=
-sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
-sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/BUILD.bazel b/BUILD.bazel
index b2443e4b5..6e09ef814 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -1,5 +1,5 @@
 exports_files([
-    "WORKSPACE.bazel",
+    "WORKSPACE.bzlmod",
 ])
 
 alias(
diff --git a/CODEOWNERS b/CODEOWNERS
index 88eebbd9a..0172574b9 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -1,65 +1,64 @@
-.golangci.yml @katexochen
-/3rdparty/gcp-guest-agent @malt3
-/bazel/ci @katexochen
-/bazel/container @katexochen
-/bazel @malt3
-/bazel/sh @katexochen
-/bootstrapper @3u13r
+# keep-sorted start
+.golangci.yml @daniel-weisse
+/3rdparty/gcp-guest-agent @msanft
+/bazel @burgerdev
+/bootstrapper @msanft
 /cli/internal/cloudcmd @daniel-weisse
-/cli/internal/cmd/upgrade* @derpsteb
+/cli/internal/cmd/upgrade* @daniel-weisse
 /cli/internal/libvirt @daniel-weisse
-/cli/internal/terraform @elchead
+/cli/internal/terraform @daniel-weisse
 /csi @daniel-weisse
-/debugd @malt3
+/debugd @daniel-weisse
 /disk-mapper @daniel-weisse
 /docs @thomasten
-/e2e @3u13r
-/hack/azure-snp-report-verify @derpsteb
-/hack/bazel-deps-mirror @malt3
+/e2e @msanft
+/hack/azure-snp-report-verify @msanft
+/hack/bazel-deps-mirror @burgerdev
+/hack/cli-k8s-compatibility @burgerdev
 /hack/clidocgen @thomasten
-/hack/cli-k8s-compatibility @derpsteb
-/hack/fetch-broken-e2e @katexochen
-/hack/gocoverage @katexochen
-/hack/oci-pin @malt3
+/hack/fetch-broken-e2e @msanft
+/hack/gocoverage @msanft
+/hack/oci-pin @burgerdev
 /hack/qemu-metadata-api @daniel-weisse
-/hack/remove-tf-providers @katexochen
-/hack/terraform @3u13r
-/hack/tools @katexochen
+/hack/remove-tf-providers @msanft
+/hack/terraform @msanft
+/hack/tools @msanft
 /hack/versioninfogen @daniel-weisse
-/image @malt3
-/internal/api @derpsteb
+/image @msanft
+/internal/api @daniel-weisse
 /internal/atls @thomasten
 /internal/attestation @daniel-weisse
-/internal/cloud @3u13r
-/internal/compatibility @derpsteb
-/internal/config @derpsteb
+/internal/cloud @msanft
+/internal/compatibility @daniel-weisse
+/internal/config @msanft
+/internal/constellation/featureset @thomasten
+/internal/constellation/helm @burgerdev
 /internal/constellation/kubecmd @daniel-weisse
-/internal/containerimage @malt3
+/internal/constellation/state @msanft
+/internal/containerimage @burgerdev
 /internal/crypto @thomasten
 /internal/cryptsetup @daniel-weisse
-/internal/constellation/featureset @malt3
 /internal/file @daniel-weisse
 /internal/grpc @thomasten
-/internal/constellation/helm @derpsteb
-/internal/imagefetcher @malt3
-/internal/installer @3u13r
+/internal/imagefetcher @msanft
+/internal/installer @msanft
 /internal/kms @daniel-weisse
-/internal/kubernetes @malt3
+/internal/kubernetes @msanft
 /internal/license @thomasten
 /internal/logger @daniel-weisse
 /internal/nodestate @daniel-weisse
-/internal/osimage @malt3
-/internal/retry @katexochen
-/internal/semver @derpsteb
-/internal/sigstore @elchead
-/internal/constellation/state @elchead
-/internal/staticupload @malt3
-/internal/versions @3u13r
+/internal/osimage @msanft
+/internal/retry @msanft
+/internal/semver @daniel-weisse
+/internal/sigstore @burgerdev
+/internal/staticupload @msanft
+/internal/versions @msanft
 /joinservice @daniel-weisse
 /keyservice @daniel-weisse
 /measurement-reader @daniel-weisse
-/operators @malt3
-/rpm @malt3
-/tools @malt3
-/upgrade-agent @3u13r
+/operators @msanft
+/terraform-provider-constellation @msanft
+/tools @burgerdev
+/upgrade-agent @msanft
 /verify @daniel-weisse
+# keep-sorted end
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9ee09c7ea..a123a5d58 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -12,7 +12,7 @@ If you want to support our development:
 * Share our projects on social media
 * Join the [Confidential Computing Discord](https://discord.gg/rH8QTH56JN)
 
-Constellation is licensed under the [AGPLv3](LICENSE). When contributing, you also need to agree to our [Contributor License Agreement](https://cla-assistant.io/edgelesssys/constellation).
+Constellation is licensed under the [BUSL](LICENSE). When contributing, you also need to agree to our [Contributor License Agreement](https://cla-assistant.io/edgelesssys/constellation).
 
 ## Reporting issues and bugs, asking questions
 
diff --git a/LICENSE b/LICENSE
index be3f7b28e..42435e491 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,661 +1,91 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
+Business Source License 1.1
 
- Copyright (C) 2007 Free Software Foundation, Inc. 
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+Parameters
 
-                            Preamble
+Licensor:             Edgeless Systems GmbH
+Licensed Work:        Constellation
+                      The Licensed Work is (c) Edgeless Systems GmbH
+Additional Use Grant: None
 
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
+Change Date:          Four years from the date a MINOR version (SemVer) is published.
 
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
+Change License:       GNU Affero General Public License Version 3 (AGPL-3.0-only)
 
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
+For information about alternative licensing arrangements for the Software,
+please visit: https://www.edgeless.systems/enterprise-support
 
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
+Notice
 
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
+License text copyright (c) 2023 MariaDB plc, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB plc.
 
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
+-----------------------------------------------------------------------------
 
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
+Business Source License 1.1
 
-  The precise terms and conditions for copying, distribution and
-modification follow.
+Terms
 
-                       TERMS AND CONDITIONS
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
 
-  0. Definitions.
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
 
-  "This License" refers to version 3 of the GNU Affero General Public License.
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
 
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
 
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
 
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
 
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
 
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
 
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
+MariaDB hereby grants you permission to use this License’s text to license
+your works, and to refer to it using the trademark “Business Source License”,
+as long as you comply with the Covenants of Licensor below.
 
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
+Covenants of Licensor
 
-  1. Source Code.
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
 
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where “compatible” means that software provided under the Change License can
+   be included in a program with software provided under GPL Version 2.0 or a
+   later version. Licensor may specify additional Change Licenses without
+   limitation.
 
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text “None”.
 
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
+3. To specify a Change Date.
 
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    
-    Copyright (C)   
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see .
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-.
+4. Not to modify this License in any other way.
diff --git a/MODULE.bazel b/MODULE.bazel
new file mode 100644
index 000000000..01097d490
--- /dev/null
+++ b/MODULE.bazel
@@ -0,0 +1,209 @@
+module(name = "constellation")
+
+bazel_dep(name = "aspect_bazel_lib", version = "2.19.4")
+
+bazel_lib = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
+bazel_lib.yq()
+use_repo(bazel_lib, "jq_toolchains")
+use_repo(bazel_lib, "yq_toolchains")
+
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
+bazel_dep(name = "gazelle", version = "0.43.0")
+bazel_dep(name = "hermetic_cc_toolchain", version = "3.2.0")
+bazel_dep(name = "rules_cc", version = "0.1.2")
+bazel_dep(name = "rules_go", version = "0.55.1", repo_name = "io_bazel_rules_go")
+bazel_dep(name = "rules_pkg", version = "1.1.0")
+bazel_dep(name = "rules_proto", version = "7.1.0")
+bazel_dep(name = "rules_python", version = "1.4.1")
+bazel_dep(name = "rules_shell", version = "0.5.0")
+
+bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True)
+
+go_sdk = use_extension("@io_bazel_rules_go//go:extensions.bzl", "go_sdk")
+go_sdk.download(
+    name = "go_sdk",
+    patches = ["//3rdparty/bazel/org_golang:go_tls_max_handshake_size.patch"],
+    version = "1.24.4",
+)
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    ignore_root_user_error = True,
+    python_version = "3.11",
+)
+
+# the use_repo rule needs to list all top-level go dependencies
+# update automatically using `bazel mod tidy`.
+go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
+go_deps.from_file(go_work = "//:go.work")
+use_repo(
+    go_deps,
+    "cat_dario_mergo",
+    "com_github_aws_aws_sdk_go",
+    "com_github_aws_aws_sdk_go_v2",
+    "com_github_aws_aws_sdk_go_v2_config",
+    "com_github_aws_aws_sdk_go_v2_credentials",
+    "com_github_aws_aws_sdk_go_v2_feature_ec2_imds",
+    "com_github_aws_aws_sdk_go_v2_feature_s3_manager",
+    "com_github_aws_aws_sdk_go_v2_service_autoscaling",
+    "com_github_aws_aws_sdk_go_v2_service_cloudfront",
+    "com_github_aws_aws_sdk_go_v2_service_ec2",
+    "com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2",
+    "com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi",
+    "com_github_aws_aws_sdk_go_v2_service_s3",
+    "com_github_aws_aws_sdk_go_v2_service_secretsmanager",
+    "com_github_aws_smithy_go",
+    "com_github_azure_azure_sdk_for_go",
+    "com_github_azure_azure_sdk_for_go_sdk_azcore",
+    "com_github_azure_azure_sdk_for_go_sdk_azidentity",
+    "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v6",
+    "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v6",
+    "com_github_azure_azure_sdk_for_go_sdk_security_keyvault_azsecrets",
+    "com_github_azure_azure_sdk_for_go_sdk_storage_azblob",
+    "com_github_bazelbuild_buildtools",
+    "com_github_burntsushi_toml",
+    "com_github_coreos_go_systemd_v22",
+    "com_github_docker_docker",
+    "com_github_edgelesssys_go_azguestattestation",
+    "com_github_edgelesssys_go_tdx_qpl",
+    "com_github_foxboron_go_uefi",
+    "com_github_fsnotify_fsnotify",
+    "com_github_go_playground_locales",
+    "com_github_go_playground_universal_translator",
+    "com_github_go_playground_validator_v10",
+    "com_github_golang_jwt_jwt_v5",
+    "com_github_google_go_licenses",
+    "com_github_google_go_sev_guest",
+    "com_github_google_go_tdx_guest",
+    "com_github_google_go_tpm",
+    "com_github_google_go_tpm_tools",
+    "com_github_google_keep_sorted",
+    "com_github_google_uuid",
+    "com_github_googleapis_gax_go_v2",
+    "com_github_gophercloud_gophercloud_v2",
+    "com_github_gophercloud_utils_v2",
+    "com_github_grpc_ecosystem_go_grpc_middleware_v2",
+    "com_github_hashicorp_go_kms_wrapping_v2",
+    "com_github_hashicorp_go_kms_wrapping_wrappers_awskms_v2",
+    "com_github_hashicorp_go_kms_wrapping_wrappers_azurekeyvault_v2",
+    "com_github_hashicorp_go_kms_wrapping_wrappers_gcpckms_v2",
+    "com_github_hashicorp_go_version",
+    "com_github_hashicorp_hc_install",
+    "com_github_hashicorp_hcl_v2",
+    "com_github_hashicorp_terraform_exec",
+    "com_github_hashicorp_terraform_json",
+    "com_github_hashicorp_terraform_plugin_framework",
+    "com_github_hashicorp_terraform_plugin_framework_validators",
+    "com_github_hashicorp_terraform_plugin_go",
+    "com_github_hashicorp_terraform_plugin_log",
+    "com_github_hashicorp_terraform_plugin_testing",
+    "com_github_hexops_gotextdiff",
+    "com_github_katexochen_sh_v3",
+    "com_github_martinjungblut_go_cryptsetup",
+    "com_github_mattn_go_isatty",
+    "com_github_mitchellh_go_homedir",
+    "com_github_onsi_ginkgo_v2",
+    "com_github_onsi_gomega",
+    "com_github_pkg_errors",
+    "com_github_regclient_regclient",
+    "com_github_rogpeppe_go_internal",
+    "com_github_samber_slog_multi",
+    "com_github_schollz_progressbar_v3",
+    "com_github_secure_systems_lab_go_securesystemslib",
+    "com_github_siderolabs_talos_pkg_machinery",
+    "com_github_sigstore_rekor",
+    "com_github_sigstore_sigstore",
+    "com_github_spf13_afero",
+    "com_github_spf13_cobra",
+    "com_github_spf13_pflag",
+    "com_github_stretchr_testify",
+    "com_github_tink_crypto_tink_go_v2",
+    "com_github_vincent_petithory_dataurl",
+    "com_google_cloud_go_compute",
+    "com_google_cloud_go_compute_metadata",
+    "com_google_cloud_go_kms",
+    "com_google_cloud_go_secretmanager",
+    "com_google_cloud_go_storage",
+    "in_gopkg_yaml_v3",
+    "io_etcd_go_etcd_api_v3",
+    "io_etcd_go_etcd_client_pkg_v3",
+    "io_etcd_go_etcd_client_v3",
+    "io_k8s_api",
+    "io_k8s_apiextensions_apiserver",
+    "io_k8s_apimachinery",
+    "io_k8s_apiserver",
+    "io_k8s_client_go",
+    "io_k8s_cluster_bootstrap",
+    "io_k8s_kubelet",
+    "io_k8s_kubernetes",
+    "io_k8s_mount_utils",
+    "io_k8s_sigs_controller_runtime",
+    "io_k8s_sigs_yaml",
+    "io_k8s_utils",
+    "org_golang_google_api",
+    "org_golang_google_grpc",
+    "org_golang_google_protobuf",
+    "org_golang_x_crypto",
+    "org_golang_x_exp",
+    "org_golang_x_mod",
+    "org_golang_x_sys",
+    "org_golang_x_text",
+    "org_golang_x_tools",
+    "org_golang_x_vuln",
+    "org_libvirt_go_libvirt",
+    "org_uber_go_goleak",
+    "sh_helm_helm_v3",
+)
+
+go_deps_with_disabled_proto_generation = [
+    "go.etcd.io/etcd/api/v3",
+    "k8s.io/apiserver",
+    "github.com/hashicorp/go-plugin",
+]
+
+[
+    go_deps.gazelle_override(
+        directives = [
+            "gazelle:go_generate_proto false",
+        ],
+        path = path,
+    )
+    for path in go_deps_with_disabled_proto_generation
+]
+
+go_deps.module_override(
+    patches = [
+        "//3rdparty/bazel/com_github_martinjungblut_go_cryptsetup:com_github_martinjungblut_go_cryptsetup.patch",
+    ],
+    path = "github.com/martinjungblut/go-cryptsetup",
+)
+go_deps.module_override(
+    patches = [
+        "//3rdparty/bazel/org_libvirt_go_libvirt:go_libvirt.patch",
+    ],
+    path = "libvirt.org/go/libvirt",
+)
+go_deps.module_override(
+    patches = [
+        "//3rdparty/bazel/com_github_cloudflare_circl:math_fp448_BUILD_bazel.patch",
+        "//3rdparty/bazel/com_github_cloudflare_circl:math_fp25519_BUILD_bazel.patch",
+        "//3rdparty/bazel/com_github_cloudflare_circl:dh_x448_BUILD_bazel.patch",
+        "//3rdparty/bazel/com_github_cloudflare_circl:dh_x25519_BUILD_bazel.patch",
+    ],
+    path = "github.com/cloudflare/circl",
+)
+go_deps.module_override(
+    patches = [
+        "//3rdparty/bazel/com_github_google_go_tpm_tools:com_github_google_go_tpm_tools.patch",
+        "//3rdparty/bazel/com_github_google_go_tpm_tools:ms_tpm_20_ref.patch",
+        "//3rdparty/bazel/com_github_google_go_tpm_tools:include.patch",
+    ],
+    path = "github.com/google/go-tpm-tools",
+)
+
+# TODO(msanft):
+# Remove once https://github.com/cncf/xds/issues/104 is resolved
+go_deps.gazelle_override(
+    build_file_generation = "on",
+    path = "github.com/cncf/xds/go",
+)
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
new file mode 100644
index 000000000..4130b9d5d
--- /dev/null
+++ b/MODULE.bazel.lock
@@ -0,0 +1,468 @@
+{
+  "lockFileVersion": 13,
+  "registryFileHashes": {
+    "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497",
+    "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2",
+    "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16",
+    "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915",
+    "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed",
+    "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/source.json": "9be551b8d4e3ef76875c0d744b5d6a504a27e3ae67bc6b28f46415fd2d2957da",
+    "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef",
+    "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862",
+    "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f",
+    "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.4/MODULE.bazel": "d39e4b18e594d81c526d7cfc513e7ecfa8ca9eb5b61488d1d790faa94b34f2d9",
+    "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.4/source.json": "506fa924e19fd8a33d617e33a17e4fce845f9ff9acb3a2aa7cf7300650698705",
+    "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838",
+    "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b",
+    "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd",
+    "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8",
+    "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d",
+    "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d",
+    "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a",
+    "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58",
+    "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920",
+    "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b",
+    "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d",
+    "https://bcr.bazel.build/modules/bazel_features/1.28.0/source.json": "16a3fc5b4483cb307643791f5a4b7365fa98d2e70da7c378cdbde55f0c0b32cf",
+    "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7",
+    "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b",
+    "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b",
+    "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953",
+    "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03",
+    "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e",
+    "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84",
+    "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8",
+    "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8",
+    "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350",
+    "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a",
+    "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0",
+    "https://bcr.bazel.build/modules/gazelle/0.43.0/MODULE.bazel": "846e1fe396eefc0f9ddad2b33e9bd364dd993fc2f42a88e31590fe0b0eefa3f0",
+    "https://bcr.bazel.build/modules/gazelle/0.43.0/source.json": "021a77f6625906d9d176e2fa351175e842622a5d45989312f2ad4924aab72df6",
+    "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb",
+    "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4",
+    "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6",
+    "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4",
+    "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f",
+    "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.2.0/MODULE.bazel": "8e7faec81c1f0fb65fe277ecfc75ea3636ce7bf848f88037fedd58e6eeacc28f",
+    "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.2.0/source.json": "67c2b76edff27c3ec449a935fc9468996d1a730b52a9a6f97c40c8a06d381630",
+    "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f",
+    "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5",
+    "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075",
+    "https://bcr.bazel.build/modules/jsoncpp/1.9.5/source.json": "4108ee5085dd2885a341c7fab149429db457b3169b86eb081fa245eadf69169d",
+    "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902",
+    "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92",
+    "https://bcr.bazel.build/modules/package_metadata/0.0.2/source.json": "e53a759a72488d2c0576f57491ef2da0cf4aab05ac0997314012495935531b73",
+    "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5",
+    "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f",
+    "https://bcr.bazel.build/modules/platforms/0.0.11/source.json": "f7e188b79ebedebfe75e9e1d098b8845226c7992b307e28e1496f23112e8fc29",
+    "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee",
+    "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37",
+    "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615",
+    "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814",
+    "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d",
+    "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc",
+    "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7",
+    "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c",
+    "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d",
+    "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c",
+    "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df",
+    "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92",
+    "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95",
+    "https://bcr.bazel.build/modules/protobuf/29.1/source.json": "04cca85dce26b895ed037d98336d860367fe09919208f2ad383f0df1aff63199",
+    "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0",
+    "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573",
+    "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858",
+    "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e",
+    "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/source.json": "be4789e951dd5301282729fe3d4938995dc4c1a81c2ff150afc9f1b0504c6022",
+    "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206",
+    "https://bcr.bazel.build/modules/re2/2023-09-01/source.json": "e044ce89c2883cd957a2969a43e79f7752f9656f6b20050b62f90ede21ec6eb4",
+    "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8",
+    "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e",
+    "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
+    "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0",
+    "https://bcr.bazel.build/modules/rules_cc/0.1.2/source.json": "53fcb09b5816c83ca60d9d7493faf3bfaf410dfc2f15deb52d6ddd146b8d43f0",
+    "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6",
+    "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8",
+    "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/source.json": "c8b1e2c717646f1702290959a3302a178fb639d987ab61d548105019f11e527e",
+    "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8",
+    "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270",
+    "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd",
+    "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0",
+    "https://bcr.bazel.build/modules/rules_go/0.55.1/MODULE.bazel": "a57a6fc59a74326c0b440d07cca209edf13c7d1a641e48cfbeab56e79f873609",
+    "https://bcr.bazel.build/modules/rules_go/0.55.1/source.json": "827a740c8959c9d20616889e7746cde4dcc6ee80d25146943627ccea0736328f",
+    "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
+    "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86",
+    "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963",
+    "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31",
+    "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a",
+    "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6",
+    "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab",
+    "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
+    "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1",
+    "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017",
+    "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939",
+    "https://bcr.bazel.build/modules/rules_java/8.5.1/source.json": "db1a77d81b059e0f84985db67a22f3f579a529a86b7997605be3d214a0abe38e",
+    "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
+    "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909",
+    "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036",
+    "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0",
+    "https://bcr.bazel.build/modules/rules_jvm_external/6.3/source.json": "6f5f5a5a4419ae4e37c35a5bb0a6ae657ed40b7abc5a5189111b47fcebe43197",
+    "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3",
+    "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5",
+    "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0",
+    "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d",
+    "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c",
+    "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb",
+    "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc",
+    "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff",
+    "https://bcr.bazel.build/modules/rules_pkg/1.1.0/MODULE.bazel": "9db8031e71b6ef32d1846106e10dd0ee2deac042bd9a2de22b4761b0c3036453",
+    "https://bcr.bazel.build/modules/rules_pkg/1.1.0/source.json": "fef768df13a92ce6067e1cd0cdc47560dace01354f1d921cfb1d632511f7d608",
+    "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06",
+    "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7",
+    "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f",
+    "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73",
+    "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2",
+    "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96",
+    "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e",
+    "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f",
+    "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7",
+    "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300",
+    "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382",
+    "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed",
+    "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58",
+    "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c",
+    "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43",
+    "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6",
+    "https://bcr.bazel.build/modules/rules_python/1.4.1/source.json": "8ec8c90c70ccacc4de8ca1b97f599e756fb59173e898ee08b733006650057c07",
+    "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c",
+    "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b",
+    "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592",
+    "https://bcr.bazel.build/modules/rules_shell/0.5.0/MODULE.bazel": "8c8447370594d45539f66858b602b0bb2cb2d3401a4ebb9ad25830c59c0f366d",
+    "https://bcr.bazel.build/modules/rules_shell/0.5.0/source.json": "3038276f07cbbdd1c432d1f80a2767e34143ffbb03cfa043f017e66adbba324c",
+    "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8",
+    "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c",
+    "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd",
+    "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c",
+    "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5",
+    "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216",
+    "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468",
+    "https://bcr.bazel.build/modules/tar.bzl/0.2.1/source.json": "600ac6ff61744667a439e7b814ae59c1f29632c3984fccf8000c64c9db8d7bb6",
+    "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43",
+    "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072",
+    "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3",
+    "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
+    "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
+    "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79",
+    "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d",
+    "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198"
+  },
+  "selectedYankedVersions": {},
+  "moduleExtensions": {
+    "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": {
+      "general": {
+        "bzlTransitiveDigest": "PjIds3feoYE8SGbbIq2SFTZy3zmxeO2tQevJZNDo7iY=",
+        "usagesDigest": "+hz7IHWN6A1oVJJWNDB6yZRG+RYhF76wAYItpAeIUIg=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "local_config_apple_cc_toolchains": {
+            "bzlFile": "@@apple_support~//crosstool:setup.bzl",
+            "ruleClassName": "_apple_cc_autoconf_toolchains",
+            "attributes": {}
+          },
+          "local_config_apple_cc": {
+            "bzlFile": "@@apple_support~//crosstool:setup.bzl",
+            "ruleClassName": "_apple_cc_autoconf",
+            "attributes": {}
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "apple_support~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@pybind11_bazel~//:python_configure.bzl%extension": {
+      "general": {
+        "bzlTransitiveDigest": "whINYge95GgPtysKDbNHQ0ZlWYdtKybHs5y2tLF+x7Q=",
+        "usagesDigest": "gNvOHVcAlwgDsNXD0amkv2CC96mnaCThPQoE44y8K+w=",
+        "recordedFileInputs": {
+          "@@pybind11_bazel~//MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e"
+        },
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "local_config_python": {
+            "bzlFile": "@@pybind11_bazel~//:python_configure.bzl",
+            "ruleClassName": "python_configure",
+            "attributes": {}
+          },
+          "pybind11": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "build_file": "@@pybind11_bazel~//:pybind11.BUILD",
+              "strip_prefix": "pybind11-2.11.1",
+              "urls": [
+                "https://github.com/pybind/pybind11/archive/v2.11.1.zip"
+              ]
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "pybind11_bazel~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@rules_fuzzing~//fuzzing/private:extensions.bzl%non_module_dependencies": {
+      "general": {
+        "bzlTransitiveDigest": "hVgJRQ3Er45/UUAgNn1Yp2Khcp/Y8WyafA2kXIYmQ5M=",
+        "usagesDigest": "YnIrdgwnf3iCLfChsltBdZ7yOJh706lpa2vww/i2pDI=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "platforms": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "urls": [
+                "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
+                "https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz"
+              ],
+              "sha256": "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74"
+            }
+          },
+          "rules_python": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "sha256": "d70cd72a7a4880f0000a6346253414825c19cdd40a28289bdf67b8e6480edff8",
+              "strip_prefix": "rules_python-0.28.0",
+              "url": "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz"
+            }
+          },
+          "bazel_skylib": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "sha256": "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94",
+              "urls": [
+                "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz",
+                "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz"
+              ]
+            }
+          },
+          "com_google_absl": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "urls": [
+                "https://github.com/abseil/abseil-cpp/archive/refs/tags/20240116.1.zip"
+              ],
+              "strip_prefix": "abseil-cpp-20240116.1",
+              "integrity": "sha256-7capMWOvWyoYbUaHF/b+I2U6XLMaHmky8KugWvfXYuk="
+            }
+          },
+          "rules_fuzzing_oss_fuzz": {
+            "bzlFile": "@@rules_fuzzing~//fuzzing/private/oss_fuzz:repository.bzl",
+            "ruleClassName": "oss_fuzz_repository",
+            "attributes": {}
+          },
+          "honggfuzz": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "build_file": "@@rules_fuzzing~//:honggfuzz.BUILD",
+              "sha256": "6b18ba13bc1f36b7b950c72d80f19ea67fbadc0ac0bb297ec89ad91f2eaa423e",
+              "url": "https://github.com/google/honggfuzz/archive/2.5.zip",
+              "strip_prefix": "honggfuzz-2.5"
+            }
+          },
+          "rules_fuzzing_jazzer": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_jar",
+            "attributes": {
+              "sha256": "ee6feb569d88962d59cb59e8a31eb9d007c82683f3ebc64955fd5b96f277eec2",
+              "url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer/0.20.1/jazzer-0.20.1.jar"
+            }
+          },
+          "rules_fuzzing_jazzer_api": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_jar",
+            "attributes": {
+              "sha256": "f5a60242bc408f7fa20fccf10d6c5c5ea1fcb3c6f44642fec5af88373ae7aa1b",
+              "url": "https://repo1.maven.org/maven2/com/code-intelligence/jazzer-api/0.20.1/jazzer-api-0.20.1.jar"
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "rules_fuzzing~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@rules_java~//java:rules_java_deps.bzl%compatibility_proxy": {
+      "general": {
+        "bzlTransitiveDigest": "KIX40nDfygEWbU+rq3nYpt3tVgTK/iO8PKh5VMBlN7M=",
+        "usagesDigest": "pwHZ+26iLgQdwvdZeA5wnAjKnNI3y6XO2VbhOTeo5h8=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "compatibility_proxy": {
+            "bzlFile": "@@rules_java~//java:rules_java_deps.bzl",
+            "ruleClassName": "_compatibility_proxy_repo_rule",
+            "attributes": {}
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "rules_java~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@rules_kotlin~//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": {
+      "general": {
+        "bzlTransitiveDigest": "fus14IFJ/1LGWWGKPH/U18VnJCoMjfDt1ckahqCnM0A=",
+        "usagesDigest": "aJF6fLy82rR95Ff5CZPAqxNoFgOMLMN5ImfBS0nhnkg=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "com_github_jetbrains_kotlin_git": {
+            "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:compiler.bzl",
+            "ruleClassName": "kotlin_compiler_git_repository",
+            "attributes": {
+              "urls": [
+                "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip"
+              ],
+              "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88"
+            }
+          },
+          "com_github_jetbrains_kotlin": {
+            "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:compiler.bzl",
+            "ruleClassName": "kotlin_capabilities_repository",
+            "attributes": {
+              "git_repository_name": "com_github_jetbrains_kotlin_git",
+              "compiler_version": "1.9.23"
+            }
+          },
+          "com_github_google_ksp": {
+            "bzlFile": "@@rules_kotlin~//src/main/starlark/core/repositories:ksp.bzl",
+            "ruleClassName": "ksp_compiler_plugin_repository",
+            "attributes": {
+              "urls": [
+                "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip"
+              ],
+              "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d",
+              "strip_version": "1.9.23-1.0.20"
+            }
+          },
+          "com_github_pinterest_ktlint": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_file",
+            "attributes": {
+              "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985",
+              "urls": [
+                "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint"
+              ],
+              "executable": true
+            }
+          },
+          "rules_android": {
+            "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl",
+            "ruleClassName": "http_archive",
+            "attributes": {
+              "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806",
+              "strip_prefix": "rules_android-0.1.1",
+              "urls": [
+                "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip"
+              ]
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "rules_kotlin~",
+            "bazel_tools",
+            "bazel_tools"
+          ]
+        ]
+      }
+    },
+    "@@rules_python~//python/uv:uv.bzl%uv": {
+      "general": {
+        "bzlTransitiveDigest": "Xpqjnjzy6zZ90Es9Wa888ZLHhn7IsNGbph/e6qoxzw8=",
+        "usagesDigest": "Tx9xMlyNlKU8Dq4pnYUJ8g2T1PHLiGoAs42AGwUIqh8=",
+        "recordedFileInputs": {},
+        "recordedDirentsInputs": {},
+        "envVariables": {},
+        "generatedRepoSpecs": {
+          "uv": {
+            "bzlFile": "@@rules_python~//python/uv/private:uv_toolchains_repo.bzl",
+            "ruleClassName": "uv_toolchains_repo",
+            "attributes": {
+              "toolchain_type": "'@@rules_python~//python/uv:uv_toolchain_type'",
+              "toolchain_names": [
+                "none"
+              ],
+              "toolchain_implementations": {
+                "none": "'@@rules_python~//python:none'"
+              },
+              "toolchain_compatible_with": {
+                "none": [
+                  "@platforms//:incompatible"
+                ]
+              },
+              "toolchain_target_settings": {}
+            }
+          }
+        },
+        "recordedRepoMappingEntries": [
+          [
+            "rules_python~",
+            "platforms",
+            "platforms"
+          ]
+        ]
+      }
+    }
+  }
+}
diff --git a/README.md b/README.md
index cd7291fa3..4f2706fe5 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,8 @@
-![Constellation](docs/static/img/banner.svg)
+![Constellation](docs/static/img/BannerConstellationanimated.svg)
 
 # Always Encrypted Kubernetes
 
 

- Constellation License Govulncheck Go Report Discord @@ -33,7 +32,7 @@ Encrypting your K8s is good for: ### 🔒 Everything always encrypted -* Runtime encryption: All nodes run inside AMD SEV-based Confidential VMs (CVMs). Support for Intel TDX will be added in the future. +* Runtime encryption: All nodes run inside Confidential VMs (CVMs) based on AMD SEV or Intel TDX. * Transparent encryption of network: All [pod-to-pod traffic is automatically encrypted][network-encryption] * Transparent encryption of storage: All writes to persistent storage are automatically encrypted. This includes [nodes' state disks][storage-encryption], [persistent volumes via CSI][csi], and [S3 object storage][s3proxy]. @@ -51,21 +50,22 @@ Encrypting your K8s is good for: * High availability with multi-master architecture and stacked etcd topology * Dynamic cluster autoscaling with verification and secure bootstrapping of new nodes -* Competitive performance ([see K-Bench comparison with AKS and GKE][performance]) +* Competitive [performance] ### 🧩 Easy to use and integrate * Constellation is a [CNCF-certified][certified] Kubernetes. It's aligned to Kubernetes' [version support policy][k8s-version-support] and will likely work with your existing workloads and tools. -* Support for Azure, GCP, and AWS. +* Support for AWS, Azure, GCP, and STACKIT. * Support for local installations with [MiniConstellation][first-steps-local]. +* Support for [Terraform][terraform-provider] ## Getting started If you're already familiar with Kubernetes, it's easy to get started with Constellation: -1. 📦 [Install the CLI][install] +1. 📦 [Install the CLI][install] or use the [Terraform provider][terraform-provider] 2. ⌨️ Create a Constellation cluster in the [cloud][first-steps] or [locally][first-steps-local] 3. 🏎️ [Run your app][examples] @@ -73,15 +73,6 @@ If you're already familiar with Kubernetes, it's easy to get started with Conste Learn more: ["Getting started with Constellation" videos series](https://www.youtube.com/playlist?list=PLEhAl3D5WVvRYxO_yI7KzmtJ7rJUyQgNu). -## Live demos - -We're running public instances of popular software on Constellation: - -* Rocket.Chat: ([blog post](https://www.edgeless.systems/resource-library/confidential-rocketchat/)) -* GitLab: ([blog post](https://www.edgeless.systems/resource-library/confidential-gitlab/)) - -These instances run on CVMs in Azure and Constellation keeps them end-to-end confidential. - ## Documentation To learn more, see the [documentation](https://docs.edgeless.systems/constellation). @@ -95,7 +86,7 @@ You may want to start with one of the following sections. * If something doesn't work, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). * Please file an [issue][github-issues] to get help or report a bug. -* Join the [Discord] to have a chat on confidential computing and Constellation. +* Join the [GitHub discussions](https://github.com/edgelesssys/constellation/discussions) if you have questions or would like to discuss an idea. * Visit our [blog](https://www.edgeless.systems/blog/) for technical deep-dives and tutorials and follow us on [LinkedIn] for news. * Edgeless Systems also offers [Enterprise Support][enterprise-support]. @@ -111,7 +102,7 @@ Refer to [`CONTRIBUTING.md`](CONTRIBUTING.md) on how to contribute. The most imp ## License -The Constellation source code is licensed under the [GNU Affero General Public License v3.0](LICENSE). Edgeless Systems provides pre-built and signed binaries and images for Constellation. You may use these free of charge to create and run services for internal consumption, evaluation purposes, or non-commercial use. You can find more information in the [license] section of the docs. +Constellation is licensed under the [Business Source License 1.1](LICENSE). You may use it free of charge for non-production use. You can find more information in the [license] section of the docs. [architecture]: https://docs.edgeless.systems/constellation/architecture/overview @@ -119,7 +110,6 @@ The Constellation source code is licensed under the [GNU Affero General Public L [cla-assistant]: https://cla-assistant.io/edgelesssys/constellation [cluster-attestation]: https://docs.edgeless.systems/constellation/architecture/attestation#cluster-attestation [confidential-kubernetes]: https://docs.edgeless.systems/constellation/overview/confidential-kubernetes -[discord]: https://discord.gg/rH8QTH56JN [enterprise-support]: https://www.edgeless.systems/products/constellation/ [first-steps]: https://docs.edgeless.systems/constellation/getting-started/first-steps [first-steps-local]: https://docs.edgeless.systems/constellation/getting-started/first-steps-local @@ -139,3 +129,4 @@ The Constellation source code is licensed under the [GNU Affero General Public L [linkedin]: https://www.linkedin.com/company/edgeless-systems [whitepaper]: https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf [performance]: https://docs.edgeless.systems/constellation/overview/performance +[terraform-provider]: https://docs.edgeless.systems/constellation/workflows/terraform-provider diff --git a/WORKSPACE.bazel b/WORKSPACE.bzlmod similarity index 72% rename from WORKSPACE.bazel rename to WORKSPACE.bzlmod index dad1f5094..273d85e75 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bzlmod @@ -1,9 +1,5 @@ workspace(name = "constellation") -load("//bazel/toolchains:skylib_deps.bzl", "skylib_deps") - -skylib_deps() - # nixpkgs deps load("//bazel/toolchains:nixpkgs_deps.bzl", "nixpkgs_deps") @@ -32,7 +28,7 @@ nixpkgs_flake_package( name = "bazel", nix_flake_file = "//:flake.nix", nix_flake_lock_file = "//:flake.lock", - package = "bazel_6", + package = "bazel_7", ) nixpkgs_flake_package( @@ -56,6 +52,21 @@ nixpkgs_flake_package( package = "mkosi", ) +nixpkgs_flake_package( + name = "uplosi", + nix_flake_file = "//:flake.nix", + nix_flake_lock_file = "//:flake.lock", + package = "uplosi", +) + +nixpkgs_flake_package( + name = "vpn_oci_image", + build_file_content = """exports_files(["layer.tar"])""", + nix_flake_file = "//:flake.nix", + nix_flake_lock_file = "//:flake.lock", + package = "vpn", +) + nixpkgs_package( name = "diffutils", repository = "@nixpkgs", @@ -96,6 +107,21 @@ nixpkgs_package( repository = "@nixpkgs", ) +nixpkgs_package( + name = "parallel", + repository = "@nixpkgs", +) + +nixpkgs_package( + name = "cosign", + repository = "@nixpkgs", +) + +nixpkgs_package( + name = "rekor-cli", + repository = "@nixpkgs", +) + load("//nix/cc:nixpkgs_cc_libraries.bzl", "nixpkgs_cc_library_deps") nixpkgs_cc_library_deps() @@ -106,15 +132,6 @@ register_mkosi( name = "mkosi_nix_toolchain", ) -# Python toolchain -load("//bazel/toolchains:python_deps.bzl", "python_deps") - -python_deps() - -load("@rules_python//python:repositories.bzl", "py_repositories") - -py_repositories() - nixpkgs_python_configure( fail_not_supported = False, python3_attribute_path = "python311", @@ -122,54 +139,11 @@ nixpkgs_python_configure( ) # Go toolchain -load("//bazel/toolchains:go_rules_deps.bzl", "go_deps") -go_deps() - -load("//bazel/toolchains:go_module_deps.bzl", "go_dependencies") - -# gazelle:repository_macro bazel/toolchains/go_module_deps.bzl%go_dependencies -go_dependencies() - -load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies") - -go_rules_dependencies() - -load("@io_tweag_rules_nixpkgs//nixpkgs:toolchains/go.bzl", "nixpkgs_go_configure") - -nixpkgs_go_configure( - nix_file = "//bazel/go:go.nix", - repository = "@nixpkgs", -) - -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") - -gazelle_dependencies(go_repository_default_config = "//:WORKSPACE.bazel") - -# gazelle:repo bazel_gazelle - -# proto toolchain -load("//bazel/toolchains:proto_deps.bzl", "proto_deps") - -proto_deps() - -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") - -rules_proto_dependencies() - -rules_proto_toolchains() - -# Buildifier -load("//bazel/toolchains:buildifier_deps.bzl", "buildifier_deps") - -buildifier_deps() +# gazelle:repo gazelle # C / C++ toolchains -load("//bazel/toolchains:hermetic_cc_deps.bzl", "hermetic_cc_deps") - -hermetic_cc_deps() - load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains") # If needed, we can specify a specific version of the Zig toolchain to use. @@ -191,6 +165,7 @@ zig_toolchains() nixpkgs_cc_configure( name = "nixpkgs_cc_toolchain", + cc_std = "c++14", # TODO(malt3): Use clang once cc-wrapper path reset bug is fixed upstream. # attribute_path = "clang_11", repository = "@nixpkgs", @@ -198,6 +173,7 @@ nixpkgs_cc_configure( nixpkgs_cc_configure( name = "nixpkgs_cc_aarch64_darwin_x86_64_linux", + cc_std = "c++14", cross_cpu = "k8", exec_constraints = [ "@platforms//os:osx", @@ -232,28 +208,6 @@ register_toolchains( "@zig_sdk//toolchain:windows_amd64", ) -# Packaging rules (tar) -load("//bazel/toolchains:pkg_deps.bzl", "pkg_deps") - -pkg_deps() - -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") - -rules_pkg_dependencies() - -# Aspect Bazel Lib -load("//bazel/toolchains:aspect_bazel_lib.bzl", "aspect_bazel_lib") - -aspect_bazel_lib() - -load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains", "register_coreutils_toolchains", "register_yq_toolchains") - -aspect_bazel_lib_dependencies() - -aspect_bazel_lib_register_toolchains() - -register_coreutils_toolchains() - # OCI rules load("//bazel/toolchains:oci_deps.bzl", "oci_deps") @@ -263,19 +217,14 @@ load("@rules_oci//oci:dependencies.bzl", "rules_oci_dependencies") rules_oci_dependencies() -load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "oci_register_toolchains") +load("@rules_oci//oci:repositories.bzl", "oci_register_toolchains") -oci_register_toolchains( - name = "oci", - crane_version = LATEST_CRANE_VERSION, -) +oci_register_toolchains(name = "oci") load("//bazel/toolchains:container_images.bzl", "containter_image_deps") containter_image_deps() -register_yq_toolchains() - # Multirun load("//bazel/toolchains:multirun_deps.bzl", "multirun_deps") @@ -285,6 +234,10 @@ load("//3rdparty/bazel/com_github_medik8s_node_maintainance_operator:source.bzl" node_maintainance_operator_deps() +load("//3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller:source.bzl", "aws_load_balancer_controller_deps") + +aws_load_balancer_controller_deps() + # CI deps load("//bazel/toolchains:ci_deps.bzl", "ci_deps") diff --git a/bazel/bazel-complete.bash b/bazel/bazel-complete.bash deleted file mode 100755 index 3c1259b44..000000000 --- a/bazel/bazel-complete.bash +++ /dev/null @@ -1,15102 +0,0 @@ -# -*- sh -*- (Bash only) -# -# Copyright 2018 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bash completion of Bazel commands. -# -# Provides command-completion for: -# - bazel prefix options (e.g. --host_jvm_args) -# - blaze command-set (e.g. build, test) -# - blaze command-specific options (e.g. --copts) -# - values for enum-valued options -# - package-names, exploring all package-path roots. -# - targets within packages. - -# Environment variables for customizing behavior: -# -# BAZEL_COMPLETION_USE_QUERY - if "true", `bazel query` will be used for -# autocompletion; otherwise, a heuristic grep is used. This is more accurate -# than the heuristic grep, especially for strangely-formatted BUILD files. But -# it can be slower, especially if the Bazel server is busy, and more brittle, if -# the BUILD file contains serious errors. This is an experimental feature. -# -# BAZEL_COMPLETION_ALLOW_TESTS_FOR_RUN - if "true", autocompletion results for -# `bazel run` includes test targets. This is convenient for users who run a lot -# of tests/benchmarks locally with 'bazel run'. - -_bazel_completion_use_query() { - _bazel__is_true "${BAZEL_COMPLETION_USE_QUERY-}" -} - -_bazel_completion_allow_tests_for_run() { - _bazel__is_true "${BAZEL_COMPLETION_ALLOW_TESTS_FOR_RUN-}" -} -# -*- sh -*- (Bash only) -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The template is expanded at build time using tables of commands/options -# derived from the bazel executable built in the same client; the expansion is -# written to bazel-complete.bash. -# -# Don't use this script directly. Generate the final script with -# bazel build //scripts:bash_completion instead. - -# This script expects a header to be prepended to it that defines the following -# nullary functions: -# -# _bazel_completion_use_query - Has a successful exit code if -# BAZEL_COMPLETION_USE_QUERY is "true". -# -# _bazel_completion_allow_tests_for_run - Has a successful exit code if -# BAZEL_COMPLETION_ALLOW_TESTS_FOR_RUN is "true". - -# The package path used by the completion routines. Unfortunately -# this isn't necessarily the same as the actual package path used by -# Bazel, but that's ok. (It's impossible for us to reliably know what -# the relevant package-path, so this is just a good guess. Users can -# override it if they want.) -: ${BAZEL_COMPLETION_PACKAGE_PATH:=%workspace%} - -# Some commands might interfer with the important one, so don't complete them -: ${BAZEL_IGNORED_COMMAND_REGEX:="__none__"} - -# bazel & ibazel commands -: ${BAZEL:=bazel} -: ${IBAZEL:=ibazel} - -# Pattern to match for looking for a target -# BAZEL_BUILD_MATCH_PATTERN__* give the pattern for label-* -# when looking in the build file. -# BAZEL_QUERY_MATCH_PATTERN__* give the pattern for label-* -# when using 'bazel query'. -# _RUNTEST is a special case for _bazel_completion_allow_tests_for_run. -: ${BAZEL_BUILD_MATCH_PATTERN__test:='(.*_test|test_suite)'} -: ${BAZEL_QUERY_MATCH_PATTERN__test:='(test|test_suite)'} -: ${BAZEL_BUILD_MATCH_PATTERN__bin:='.*_binary'} -: ${BAZEL_QUERY_MATCH_PATTERN__bin:='(binary)'} -: ${BAZEL_BUILD_MATCH_PATTERN_RUNTEST__bin:='(.*_(binary|test)|test_suite)'} -: ${BAZEL_QUERY_MATCH_PATTERN_RUNTEST__bin:='(binary|test)'} -: ${BAZEL_BUILD_MATCH_PATTERN__:='.*'} -: ${BAZEL_QUERY_MATCH_PATTERN__:=''} - -# Usage: _bazel__get_rule_match_pattern -# Determine what kind of rules to match, based on command. -_bazel__get_rule_match_pattern() { - local var_name pattern - if _bazel_completion_use_query; then - var_name="BAZEL_QUERY_MATCH_PATTERN" - else - var_name="BAZEL_BUILD_MATCH_PATTERN" - fi - if [[ $1 =~ ^label-?([a-z]*)$ ]]; then - pattern=${BASH_REMATCH[1]-} - if _bazel_completion_allow_tests_for_run; then - eval "echo \"\${${var_name}_RUNTEST__${pattern}:-\$${var_name}__${pattern}}\"" - else - eval "echo \"\$${var_name}__${pattern}\"" - fi - fi -} - -# Compute workspace directory. Search for the innermost -# enclosing directory with a WORKSPACE file. -_bazel__get_workspace_path() { - local workspace=$PWD - while true; do - if [ -f "${workspace}/WORKSPACE" ]; then - break - elif [ -z "$workspace" -o "$workspace" = "/" ]; then - workspace=$PWD - break - fi - workspace=${workspace%/*} - done - echo $workspace -} - -# Find the current piece of the line to complete, but only do word breaks at -# certain characters. In particular, ignore these: "':= -# This method also takes into account the current cursor position. -# -# Works with both bash 3 and 4! Bash 3 and 4 perform different word breaks when -# computing the COMP_WORDS array. We need this here because Bazel options are of -# the form --a=b, and labels of the form //some/label:target. -_bazel__get_cword() { - local cur=${COMP_LINE:0:COMP_POINT} - # This expression finds the last word break character, as defined in the - # COMP_WORDBREAKS variable, but without '=' or ':', which is not preceeded by - # a slash. Quote characters are also excluded. - local wordbreaks="$COMP_WORDBREAKS" - wordbreaks="${wordbreaks//\'/}" - wordbreaks="${wordbreaks//\"/}" - wordbreaks="${wordbreaks//:/}" - wordbreaks="${wordbreaks//=/}" - local word_start=$(expr "$cur" : '.*[^\]['"${wordbreaks}"']') - echo "${cur:word_start}" -} - -# Usage: _bazel__package_path -# -# Prints a list of package-path root directories, displaced using the -# current displacement from the workspace. All elements have a -# trailing slash. -_bazel__package_path() { - local workspace=$1 displacement=$2 root - IFS=: - for root in ${BAZEL_COMPLETION_PACKAGE_PATH//\%workspace\%/$workspace}; do - unset IFS - echo "$root/$displacement" - done -} - -# Usage: _bazel__options_for -# -# Prints the set of options for a given Bazel command, e.g. "build". -_bazel__options_for() { - local options - if [[ ${BAZEL_COMMAND_LIST} =~ ^(.* )?$1( .*)?$ ]]; then - # assumes option names only use ASCII characters - local option_name=$(echo $1 | tr a-z A-Z | tr "-" "_") - eval "echo \${BAZEL_COMMAND_${option_name}_FLAGS}" | tr " " "\n" - fi -} -# Usage: _bazel__expansion_for -# -# Prints the completion pattern for a given Bazel command, e.g. "build". -_bazel__expansion_for() { - local options - if [[ ${BAZEL_COMMAND_LIST} =~ ^(.* )?$1( .*)?$ ]]; then - # assumes option names only use ASCII characters - local option_name=$(echo $1 | tr a-z A-Z | tr "-" "_") - eval "echo \${BAZEL_COMMAND_${option_name}_ARGUMENT}" - fi -} - -# Usage: _bazel__matching_targets -# -# Prints target names of kind and starting with in the BUILD -# file given as standard input. is a basic regex (BRE) used to match the -# bazel rule kind and is the prefix of the target name. -_bazel__matching_targets() { - local kind_pattern="$1" - local target_prefix="$2" - # The following commands do respectively: - # Remove BUILD file comments - # Replace \n by spaces to have the BUILD file in a single line - # Extract all rule types and target names - # Grep the kind pattern and the target prefix - # Returns the target name - sed 's/#.*$//' | - tr "\n" " " | - sed 's/\([a-zA-Z0-9_]*\) *(\([^)]* \)\{0,1\}name *= *['\''"]\([a-zA-Z0-9_/.+=,@~-]*\)['\''"][^)]*)/\ -type:\1 name:\3\ -/g' | - "grep" -E "^type:$kind_pattern name:$target_prefix" | - cut -d ':' -f 3 -} - -# Usage: _bazel__is_true -# -# Returns true or false based on the input string. The following are -# valid true values (the rest are false): "1", "true". -_bazel__is_true() { - local str="$1" - [[ $str == "1" || $str == "true" ]] -} - -# Usage: _bazel__expand_rules_in_package -# -# -# Expands rules in specified packages, exploring all roots of -# $BAZEL_COMPLETION_PACKAGE_PATH, not just $(pwd). Only rules -# appropriate to the command are printed. Sets $COMPREPLY array to -# result. -# -# If _bazel_completion_use_query has a successful exit code, 'bazel query' is -# used instead, with the actual Bazel package path; -# $BAZEL_COMPLETION_PACKAGE_PATH is ignored in this case, since the actual Bazel -# value is likely to be more accurate. -_bazel__expand_rules_in_package() { - local workspace=$1 displacement=$2 current=$3 label_type=$4 - local package_name=$(echo "$current" | cut -f1 -d:) - local rule_prefix=$(echo "$current" | cut -f2 -d:) - local root buildfile rule_pattern r result - - result= - pattern=$(_bazel__get_rule_match_pattern "$label_type") - if _bazel_completion_use_query; then - package_name=$(echo "$package_name" | tr -d "'\"") # remove quotes - result=$(${BAZEL} --output_base=/tmp/${BAZEL}-completion-$USER query \ - --keep_going --noshow_progress --output=label \ - "kind('$pattern rule', '$package_name:*')" 2> /dev/null | - cut -f2 -d: | "grep" "^$rule_prefix") - else - for root in $(_bazel__package_path "$workspace" "$displacement"); do - buildfile="$root/$package_name/BUILD.bazel" - if [ ! -f "$buildfile" ]; then - buildfile="$root/$package_name/BUILD" - fi - if [ -f "$buildfile" ]; then - result=$(_bazel__matching_targets \ - "$pattern" "$rule_prefix" < "$buildfile") - break - fi - done - fi - - index=$(echo $result | wc -w) - if [ -n "$result" ]; then - echo "$result" | tr " " "\n" | sed 's|$| |' - fi - # Include ":all" wildcard if there was no unique match. (The zero - # case is tricky: we need to include "all" in that case since - # otherwise we won't expand "a" to "all" in the absence of rules - # starting with "a".) - if [ $index -ne 1 ] && expr all : "\\($rule_prefix\\)" > /dev/null; then - echo "all " - fi -} - -# Usage: _bazel__expand_package_name -# -# -# Expands directories, but explores all roots of -# BAZEL_COMPLETION_PACKAGE_PATH, not just $(pwd). When a directory is -# a bazel package, the completion offers "pkg:" so you can expand -# inside the package. -# Sets $COMPREPLY array to result. -_bazel__expand_package_name() { - local workspace=$1 displacement=$2 current=$3 type=${4-} root dir index - for root in $(_bazel__package_path "$workspace" "$displacement"); do - found=0 - for dir in $(compgen -d $root$current); do - [ -L "$dir" ] && continue # skip symlinks (e.g. bazel-bin) - [[ $dir =~ ^(.*/)?\.[^/]*$ ]] && continue # skip dotted dir (e.g. .git) - found=1 - echo "${dir#$root}/" - if [ -f $dir/BUILD.bazel -o -f $dir/BUILD ]; then - if [ "${type}" = "label-package" ]; then - echo "${dir#$root} " - else - echo "${dir#$root}:" - fi - fi - done - [ $found -gt 0 ] && break # Stop searching package path upon first match. - done -} - -# Usage: _bazel__expand_target_pattern -# -# -# Expands "word" to match target patterns, using the current workspace -# and displacement from it. "command" is used to filter rules. -# Sets $COMPREPLY array to result. -_bazel__expand_target_pattern() { - local workspace=$1 displacement=$2 current=$3 label_syntax=$4 - case "$current" in - //*:*) # Expand rule names within package, no displacement. - if [ "${label_syntax}" = "label-package" ]; then - compgen -S " " -W "BUILD" "$(echo current | cut -f ':' -d2)" - else - _bazel__expand_rules_in_package "$workspace" "" "$current" "$label_syntax" - fi - ;; - *:*) # Expand rule names within package, displaced. - if [ "${label_syntax}" = "label-package" ]; then - compgen -S " " -W "BUILD" "$(echo current | cut -f ':' -d2)" - else - _bazel__expand_rules_in_package \ - "$workspace" "$displacement" "$current" "$label_syntax" - fi - ;; - //*) # Expand filenames using package-path, no displacement - _bazel__expand_package_name "$workspace" "" "$current" "$label_syntax" - ;; - *) # Expand filenames using package-path, displaced. - if [ -n "$current" ]; then - _bazel__expand_package_name "$workspace" "$displacement" "$current" "$label_syntax" - fi - ;; - esac -} - -_bazel__get_command() { - for word in "${COMP_WORDS[@]:1:COMP_CWORD-1}"; do - if echo "$BAZEL_COMMAND_LIST" | "grep" -wsq -e "$word"; then - echo $word - break - fi - done -} - -# Returns the displacement to the workspace given in $1 -_bazel__get_displacement() { - if [[ $PWD =~ ^$1/.*$ ]]; then - echo ${PWD##$1/}/ - fi -} - -# Usage: _bazel__complete_pattern -# -# -# Expand a word according to a type. The currently supported types are: -# - {a,b,c}: an enum that can take value a, b or c -# - label: a label of any kind -# - label-bin: a label to a runnable rule (basically to a _binary rule) -# - label-test: a label to a test rule -# - info-key: an info key as listed by `bazel help info-keys` -# - command: the name of a command -# - path: a file path -# - combinaison of previous type using | as separator -_bazel__complete_pattern() { - local workspace=$1 displacement=$2 current=$3 types=$4 - for type in $(echo $types | tr "|" "\n"); do - case "$type" in - label*) - _bazel__expand_target_pattern "$workspace" "$displacement" \ - "$current" "$type" - ;; - info-key) - compgen -S " " -W "${BAZEL_INFO_KEYS}" -- "$current" - ;; - "command") - local commands=$(echo "${BAZEL_COMMAND_LIST}" | - tr " " "\n" | "grep" -v "^${BAZEL_IGNORED_COMMAND_REGEX}$") - compgen -S " " -W "${commands}" -- "$current" - ;; - path) - for file in $(compgen -f -- "$current"); do - if [[ -d $file ]]; then - echo "$file/" - else - echo "$file " - fi - done - ;; - *) - compgen -S " " -W "$type" -- "$current" - ;; - esac - done -} - -# Usage: _bazel__expand_options -# -# -# Expands options, making sure that if current-word contains an equals sign, -# it is handled appropriately. -_bazel__expand_options() { - local workspace="$1" displacement="$2" cur="$3" options="$4" - if [[ $cur =~ = ]]; then - # also expands special labels - current=$(echo "$cur" | cut -f2 -d=) - _bazel__complete_pattern "$workspace" "$displacement" "$current" \ - "$(compgen -W "$options" -- "$cur" | cut -f2 -d=)" | - sort -u - else - compgen -W "$(echo "$options" | sed 's|=.*$|=|')" -- "$cur" | - sed 's|\([^=]\)$|\1 |' - fi -} - -# Usage: _bazel__abspath -# -# -# Returns the absolute path to a file -_bazel__abspath() { - echo "$( - cd "$(dirname "$1")" - pwd - )/$(basename "$1")" -} - -# Usage: _bazel__rc_imports -# -# -# Returns the list of other RC imported (or try-imported) by a given RC file -# Only return files we can actually find, and only return absolute paths -_bazel__rc_imports() { - local workspace="$1" rc_dir rc_file="$2" import_files - rc_dir=$(dirname $rc_file) - import_files=$(cat $rc_file | - sed 's/#.*//' | - sed -E "/^(try-){0,1}import/!d" | - sed -E "s/^(try-){0,1}import ([^ ]*).*$/\2/" | - sort -u) - - local confirmed_import_files=() - for import in $import_files; do - # rc imports can use %workspace% to refer to the workspace, and we need to substitute that here - import=${import//\%workspace\%/$workspace} - if [[ ${import:0:1} != "/" ]]; then - import="$rc_dir/$import" - fi - import=$(_bazel__abspath $import) - # Don't bother dealing with it further if we can't find it - if [ -f "$import" ]; then - confirmed_import_files+=($import) - fi - done - echo "${confirmed_import_files[@]}" -} - -# Usage: _bazel__rc_expand_imports __new__ -# -# -# Function that receives a workspace and two lists. The first list is a list of RC files that have -# already been processed, and the second list contains new RC files that need processing. Each new file will be -# processed for "{try-}import" lines to discover more RC files that need parsing. -# Any lines we find in "{try-}import" will be checked against known files (and not processed again if known). -_bazel__rc_expand_imports() { - local workspace="$1" rc_file new_found="no" processed_files=() to_process_files=() discovered_files=() - # We've consumed workspace - shift - # Now grab everything else - local all_files=($@) - for rc_file in ${all_files[@]}; do - if [ "$rc_file" == "__new__" ]; then - new_found="yes" - continue - elif [ "$new_found" == "no" ]; then - processed_files+=($rc_file) - else - to_process_files+=($rc_file) - fi - done - - # For all the non-processed files, get the list of imports out of each of those files - for rc_file in "${to_process_files[@]}"; do - local potential_new_files+=($(_bazel__rc_imports "$workspace" "$rc_file")) - processed_files+=($rc_file) - for potential_new_file in ${potential_new_files[@]}; do - if [[ ! " ${processed_files[@]} " =~ " ${potential_new_file} " ]]; then - discovered_files+=($potential_new_file) - fi - done - done - - # Finally, return two lists (separated by __new__) of the files that have been fully processed, and - # the files that need processing. - echo "${processed_files[@]}" "__new__" "${discovered_files[@]}" -} - -# Usage: _bazel__rc_files -# -# -# Returns the list of RC files to examine, given the current command-line args. -_bazel__rc_files() { - local workspace="$1" new_files=() processed_files=() - # Handle the workspace RC unless --noworkspace_rc says not to. - if [[ ! ${COMP_LINE} =~ "--noworkspace_rc" ]]; then - local workspacerc="$workspace/.bazelrc" - if [ -f "$workspacerc" ]; then - new_files+=($(_bazel__abspath $workspacerc)) - fi - fi - # Handle the $HOME RC unless --nohome_rc says not to. - if [[ ! ${COMP_LINE} =~ "--nohome_rc" ]]; then - local homerc="$HOME/.bazelrc" - if [ -f "$homerc" ]; then - new_files+=($(_bazel__abspath $homerc)) - fi - fi - # Handle the system level RC unless --nosystem_rc says not to. - if [[ ! ${COMP_LINE} =~ "--nosystem_rc" ]]; then - local systemrc="/etc/bazel.bazelrc" - if [ -f "$systemrc" ]; then - new_files+=($(_bazel__abspath $systemrc)) - fi - fi - # Finally, if the user specified any on the command-line, then grab those - # keeping in mind that there may be several. - if [[ ${COMP_LINE} =~ "--bazelrc=" ]]; then - # There's got to be a better way to do this, but... it gets the job done, - # even if there are multiple --bazelrc on the command line. The sed command - # SHOULD be simpler, but capturing several instances of the same pattern - # from the same line is tricky because of the greedy nature of .* - # Instead we transform it to multiple lines, and then back. - local cmdlnrcs=$(echo ${COMP_LINE} | sed -E "s/--bazelrc=/\n--bazelrc=/g" | sed -E "/--bazelrc/!d;s/^--bazelrc=([^ ]*).*$/\1/g" | tr "\n" " ") - for rc_file in $cmdlnrcs; do - if [ -f "$rc_file" ]; then - new_files+=($(_bazel__abspath $rc_file)) - fi - done - fi - - # Each time we call _bazel__rc_expand_imports, it may find new files which then need to be expanded as well, - # so we loop until we've processed all new files. - while ((${#new_files[@]} > 0)); do - local all_files=($(_bazel__rc_expand_imports "$workspace" "${processed_files[@]}" "__new__" "${new_files[@]}")) - local new_found="no" - new_files=() - processed_files=() - for file in ${all_files[@]}; do - if [ "$file" == "__new__" ]; then - new_found="yes" - continue - elif [ "$new_found" == "no" ]; then - processed_files+=($file) - else - new_files+=($file) - fi - done - done - - echo "${processed_files[@]}" -} - -# Usage: _bazel__all_configs -# -# -# Gets contents of all RC files and searches them for config names -# that could be used for expansion. -_bazel__all_configs() { - local workspace="$1" command="$2" rc_files - - # Start out getting a list of all RC files that we can look for configs in - # This respects the various command line options documented at - # https://bazel.build/docs/bazelrc - rc_files=$(_bazel__rc_files "$workspace") - - # Commands can inherit configs from other commands, so build up command_match, which is - # a match list of the various commands that we can match against, given the command - # specified by the user - local build_inherit=("aquery" "clean" "coverage" "cquery" "info" "mobile-install" "print_action" "run" "test") - local test_inherit=("coverage") - local command_match="$command" - if [[ ${build_inherit[@]} =~ $command ]]; then - command_match="$command_match|build" - fi - if [[ ${test_inherit[@]} =~ $command ]]; then - command_match="$command_match|test" - fi - - # The following commands do respectively: - # Gets the contents of all relevant/allowed RC files - # Remove file comments - # Filter only the configs relevant to the current command - # Extract the config names - # Filters out redundant names and returns the results - cat $rc_files | - sed 's/#.*//' | - sed -E "/^($command_match):/!d" | - sed -E "s/^($command_match):([^ ]*).*$/\2/" | - sort -u -} - -# Usage: _bazel__expand_config -# -# -# Expands configs, checking through the allowed rc files and parsing for configs -# relevant to the current command -_bazel__expand_config() { - local workspace="$1" command="$2" cur="$3" rc_files all_configs - all_configs=$(_bazel__all_configs "$workspace" "$command") - compgen -S " " -W "$all_configs" -- "$cur" -} - -_bazel__is_after_doubledash() { - for word in "${COMP_WORDS[@]:1:COMP_CWORD-1}"; do - if [[ $word == "--" ]]; then - return 0 - fi - done - return 1 -} - -_bazel__complete_stdout() { - local cur=$(_bazel__get_cword) word command displacement workspace - - # Determine command: "" (startup-options) or one of $BAZEL_COMMAND_LIST. - command="$(_bazel__get_command)" - - workspace="$(_bazel__get_workspace_path)" - displacement="$(_bazel__get_displacement ${workspace})" - - if _bazel__is_after_doubledash && [[ $command == "run" ]]; then - _bazel__complete_pattern "$workspace" "$displacement" "${cur#*=}" "path" - else - case "$command" in - "") # Expand startup-options or commands - local commands=$(echo "${BAZEL_COMMAND_LIST}" | - tr " " "\n" | "grep" -v "^${BAZEL_IGNORED_COMMAND_REGEX}$") - _bazel__expand_options "$workspace" "$displacement" "$cur" \ - "${commands}\ - ${BAZEL_STARTUP_OPTIONS}" - ;; - - *) - case "$cur" in - --config=*) # Expand options: - _bazel__expand_config "$workspace" "$command" "${cur#"--config="}" - ;; - -*) # Expand options: - _bazel__expand_options "$workspace" "$displacement" "$cur" \ - "$(_bazel__options_for $command)" - ;; - *) # Expand target pattern - expansion_pattern="$(_bazel__expansion_for $command)" - NON_QUOTE_REGEX="^[\"']" - if [[ $command == query && $cur =~ $NON_QUOTE_REGEX ]]; then - : # Ideally we would expand query expressions---it's not - # that hard, conceptually---but readline is just too - # damn complex when it comes to quotation. Instead, - # for query, we just expand target patterns, unless - # the first char is a quote. - elif [ -n "$expansion_pattern" ]; then - _bazel__complete_pattern \ - "$workspace" "$displacement" "$cur" "$expansion_pattern" - fi - ;; - esac - ;; - esac - fi -} - -_bazel__to_compreply() { - local replies="$1" - COMPREPLY=() - # Trick to preserve whitespaces - while IFS="" read -r reply; do - COMPREPLY+=("${reply}") - done < <(echo "${replies}") - # Null may be set despite there being no completions - if [ ${#COMPREPLY[@]} -eq 1 ] && [ -z ${COMPREPLY[0]} ]; then - COMPREPLY=() - fi -} - -_bazel__complete() { - _bazel__to_compreply "$(_bazel__complete_stdout)" -} - -# Some users have aliases such as bt="bazel test" or bb="bazel build", this -# completion function allows them to have auto-completion for these aliases. -_bazel__complete_target_stdout() { - local cur=$(_bazel__get_cword) word command displacement workspace - - # Determine command: "" (startup-options) or one of $BAZEL_COMMAND_LIST. - command="$1" - - workspace="$(_bazel__get_workspace_path)" - displacement="$(_bazel__get_displacement ${workspace})" - - _bazel__to_compreply "$(_bazel__expand_target_pattern "$workspace" "$displacement" \ - "$cur" "$(_bazel__expansion_for $command)")" -} - -# default completion for bazel -complete -F _bazel__complete -o nospace "${BAZEL}" -complete -F _bazel__complete -o nospace "${IBAZEL}" -BAZEL_COMMAND_LIST="analyze-profile aquery build canonicalize-flags clean config coverage cquery dump fetch help info license mobile-install modquery print_action query run shutdown sync test version" -BAZEL_INFO_KEYS=" -workspace -install_base -output_base -execution_root -output_path -client-env -bazel-bin -bazel-genfiles -bazel-testlogs -release -server_pid -server_log -package_path -used-heap-size -used-heap-size-after-gc -committed-heap-size -max-heap-size -gc-time -gc-count -java-runtime -java-vm -java-home -character-encoding -defaults-package -build-language -default-package-path -starlark-semantics -worker_metrics -local_resources -" -BAZEL_STARTUP_OPTIONS=" ---autodetect_server_javabase ---noautodetect_server_javabase ---batch ---nobatch ---batch_cpu_scheduling ---nobatch_cpu_scheduling ---bazelrc= ---block_for_lock ---noblock_for_lock ---client_debug ---noclient_debug ---connect_timeout_secs= ---expand_configs_in_place ---noexpand_configs_in_place ---failure_detail_out=path ---home_rc ---nohome_rc ---host_jvm_args= ---host_jvm_debug ---host_jvm_profile= ---idle_server_tasks ---noidle_server_tasks ---ignore_all_rc_files ---noignore_all_rc_files ---io_nice_level= ---local_startup_timeout_secs= ---macos_qos_class= ---max_idle_secs= ---output_base=path ---output_user_root=path ---preemptible ---nopreemptible ---server_javabase= ---server_jvm_out=path ---shutdown_on_low_sys_mem ---noshutdown_on_low_sys_mem ---system_rc ---nosystem_rc ---unlimit_coredumps ---nounlimit_coredumps ---watchfs ---nowatchfs ---windows_enable_symlinks ---nowindows_enable_symlinks ---workspace_rc ---noworkspace_rc -" -BAZEL_COMMAND_ANALYZE_PROFILE_ARGUMENT="path" -BAZEL_COMMAND_ANALYZE_PROFILE_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---disk_cache=path ---distdir= ---dump= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_AQUERY_ARGUMENT="label" -BAZEL_COMMAND_AQUERY_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspect_deps={off,conservative,precise} ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---deduplicate_depsets ---nodeduplicate_depsets ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_parallel_aquery_output ---noexperimental_parallel_aquery_output ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---graph:factored ---nograph:factored ---graph:node_limit= ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---implicit_deps ---noimplicit_deps ---include_artifacts ---noinclude_artifacts ---include_aspects ---noinclude_aspects ---include_commandline ---noinclude_commandline ---include_file_write_contents ---noinclude_file_write_contents ---include_param_files ---noinclude_param_files ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_display_source_file_location ---noincompatible_display_source_file_location ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_package_group_includes_double_slash ---noincompatible_package_group_includes_double_slash ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---infer_universe_scope ---noinfer_universe_scope ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---line_terminator_null ---noline_terminator_null ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---nodep_deps ---nonodep_deps ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto:default_values ---noproto:default_values ---proto:definition_stack ---noproto:definition_stack ---proto:flatten_selects ---noproto:flatten_selects ---proto:include_synthetic_attribute_hash ---noproto:include_synthetic_attribute_hash ---proto:instantiation_stack ---noproto:instantiation_stack ---proto:locations ---noproto:locations ---proto:output_rule_attrs= ---proto:rule_inputs_and_outputs ---noproto:rule_inputs_and_outputs ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---query_file= ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---relative_locations ---norelative_locations ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---skyframe_state ---noskyframe_state ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_deps ---notool_deps ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---universe_scope= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_BUILD_ARGUMENT="label" -BAZEL_COMMAND_BUILD_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_CANONICALIZE_FLAGS_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---canonicalize_policy ---nocanonicalize_policy ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---for_command= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---invocation_policy= ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---show_warnings ---noshow_warnings ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_CLEAN_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---async ---noasync ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---expunge ---noexpunge ---expunge_async ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---remove_all_convenience_symlinks ---noremove_all_convenience_symlinks ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_CONFIG_ARGUMENT="string" -BAZEL_COMMAND_CONFIG_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dump_all ---nodump_all ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output={text,json} ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_COVERAGE_ARGUMENT="label-test" -BAZEL_COMMAND_COVERAGE_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---print_relative_test_log_paths ---noprint_relative_test_log_paths ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---test_verbose_timeout_warnings ---notest_verbose_timeout_warnings ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---verbose_test_summary ---noverbose_test_summary ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_CQUERY_ARGUMENT="label" -BAZEL_COMMAND_CQUERY_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspect_deps={off,conservative,precise} ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---graph:factored ---nograph:factored ---graph:node_limit= ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---implicit_deps ---noimplicit_deps ---include_aspects ---noinclude_aspects ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_display_source_file_location ---noincompatible_display_source_file_location ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_package_group_includes_double_slash ---noincompatible_package_group_includes_double_slash ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---infer_universe_scope ---noinfer_universe_scope ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---line_terminator_null ---noline_terminator_null ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---nodep_deps ---nonodep_deps ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---print_relative_test_log_paths ---noprint_relative_test_log_paths ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto:default_values ---noproto:default_values ---proto:definition_stack ---noproto:definition_stack ---proto:flatten_selects ---noproto:flatten_selects ---proto:include_configurations ---noproto:include_configurations ---proto:include_synthetic_attribute_hash ---noproto:include_synthetic_attribute_hash ---proto:instantiation_stack ---noproto:instantiation_stack ---proto:locations ---noproto:locations ---proto:output_rule_attrs= ---proto:rule_inputs_and_outputs ---noproto:rule_inputs_and_outputs ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---query_file= ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---relative_locations ---norelative_locations ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_config_fragments={off,direct,transitive} ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark:expr= ---starlark:file= ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---test_verbose_timeout_warnings ---notest_verbose_timeout_warnings ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_deps ---notool_deps ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---transitions={full,lite,none} ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---universe_scope= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---verbose_test_summary ---noverbose_test_summary ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_DUMP_FLAGS=" ---action_cache ---noaction_cache ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---packages ---nopackages ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---rule_classes ---norule_classes ---rules ---norules ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe={off,summary,count,deps,rdeps} ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---skykey_filter= ---skylark_memory= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_FETCH_ARGUMENT="label" -BAZEL_COMMAND_FETCH_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---deleted_packages= ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---loading_phase_threads= ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---package_path= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_HELP_ARGUMENT="command|{startup_options,target-syntax,info-keys}" -BAZEL_COMMAND_HELP_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---help_verbosity={long,medium,short} ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---lockfile_mode={off,update,error} ---logging= ---long ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---short ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_INFO_ARGUMENT="info-key" -BAZEL_COMMAND_INFO_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_make_env ---noshow_make_env ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_LICENSE_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_MOBILE_INSTALL_ARGUMENT="label" -BAZEL_COMMAND_MOBILE_INSTALL_FLAGS=" ---action_env= ---adb= ---adb_arg= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_app ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device= ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental ---noincremental ---incremental_dexing ---noincremental_dexing ---incremental_install_verbosity= ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---mode={classic,classic_internal_test_do_not_use,skylark} ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---split_apks ---nosplit_apks ---stamp ---nostamp ---starlark_cpu_profile= ---start={no,cold,warm,debug} ---start_app ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_MODQUERY_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---charset={utf8,ascii} ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---cycles ---nocycles ---deleted_packages= ---depth= ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---extra ---noextra ---from= ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---include_unused ---noinclude_unused ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---loading_phase_threads= ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---output={text,json,graph} ---override_module= ---override_repository= ---package_path= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_PRINT_ACTION_ARGUMENT="label" -BAZEL_COMMAND_PRINT_ACTION_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---print_action_mnemonics= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_QUERY_ARGUMENT="label" -BAZEL_COMMAND_QUERY_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---aspect_deps={off,conservative,precise} ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---deleted_packages= ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_graphless_query={auto,yes,no} ---noexperimental_graphless_query ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---graph:conditional_edges_limit= ---graph:factored ---nograph:factored ---graph:node_limit= ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---implicit_deps ---noimplicit_deps ---include_aspects ---noinclude_aspects ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_display_source_file_location ---noincompatible_display_source_file_location ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_lexicographical_output ---noincompatible_lexicographical_output ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_package_group_includes_double_slash ---noincompatible_package_group_includes_double_slash ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---infer_universe_scope ---noinfer_universe_scope ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---line_terminator_null ---noline_terminator_null ---loading_phase_threads= ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---nodep_deps ---nonodep_deps ---noorder_results ---null ---order_output={no,deps,auto,full} ---order_results ---output= ---override_module= ---override_repository= ---package_path= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---proto:default_values ---noproto:default_values ---proto:definition_stack ---noproto:definition_stack ---proto:flatten_selects ---noproto:flatten_selects ---proto:include_synthetic_attribute_hash ---noproto:include_synthetic_attribute_hash ---proto:instantiation_stack ---noproto:instantiation_stack ---proto:locations ---noproto:locations ---proto:output_rule_attrs= ---proto:rule_inputs_and_outputs ---noproto:rule_inputs_and_outputs ---query_file= ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---relative_locations ---norelative_locations ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---strict_test_suite ---nostrict_test_suite ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_deps ---notool_deps ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---universe_scope= ---watchfs ---nowatchfs ---xml:default_values ---noxml:default_values ---xml:line_numbers ---noxml:line_numbers -" -BAZEL_COMMAND_RUN_ARGUMENT="label-bin" -BAZEL_COMMAND_RUN_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---script_path=path ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_SHUTDOWN_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---iff_heap_size_greater_than= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_SYNC_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---configure ---noconfigure ---curses={yes,no,auto} ---deleted_packages= ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---loading_phase_threads= ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---only= ---override_module= ---override_repository= ---package_path= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" -BAZEL_COMMAND_TEST_ARGUMENT="label-test" -BAZEL_COMMAND_TEST_FLAGS=" ---action_env= ---allow_analysis_failures ---noallow_analysis_failures ---allow_yanked_versions= ---analysis_testing_deps_limit= ---android_compiler= ---android_cpu= ---android_crosstool_top=label ---android_databinding_use_androidx ---noandroid_databinding_use_androidx ---android_databinding_use_v3_4_args ---noandroid_databinding_use_v3_4_args ---android_dynamic_mode={off,default,fully} ---android_grte_top=label ---android_manifest_merger={legacy,android,force_android} ---android_manifest_merger_order={alphabetical,alphabetical_by_configuration,dependency} ---android_platforms= ---android_resource_shrinking ---noandroid_resource_shrinking ---android_sdk=label ---announce ---noannounce ---announce_rc ---noannounce_rc ---apk_signing_method={v1,v2,v1_v2,v4} ---apple_compiler= ---apple_crosstool_top=label ---apple_enable_auto_dsym_dbg ---noapple_enable_auto_dsym_dbg ---apple_generate_dsym ---noapple_generate_dsym ---apple_grte_top=label ---aspects= ---aspects_parameters= ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---auto_cpu_environment_group=label ---auto_output_filter={none,all,packages,subpackages} ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---break_build_on_parallel_dex2oat_failure ---nobreak_build_on_parallel_dex2oat_failure ---build ---nobuild ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_manual_tests ---nobuild_manual_tests ---build_metadata= ---build_python_zip={auto,yes,no} ---nobuild_python_zip ---build_runfile_links ---nobuild_runfile_links ---build_runfile_manifests ---nobuild_runfile_manifests ---build_tag_filters= ---build_test_dwp ---nobuild_test_dwp ---build_tests_only ---nobuild_tests_only ---cache_test_results={auto,yes,no} ---nocache_test_results ---catalyst_cpus= ---cc_output_directory_tag= ---cc_proto_library_header_suffixes= ---cc_proto_library_source_suffixes= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---check_licenses ---nocheck_licenses ---check_tests_up_to_date ---nocheck_tests_up_to_date ---check_up_to_date ---nocheck_up_to_date ---check_visibility ---nocheck_visibility ---collapse_duplicate_defines ---nocollapse_duplicate_defines ---collect_code_coverage ---nocollect_code_coverage ---color={yes,no,auto} ---combined_report={none,lcov} ---compilation_mode={fastbuild,dbg,opt} ---compile_one_dependency ---nocompile_one_dependency ---compiler= ---config= ---conlyopt= ---copt= ---coverage_output_generator=label ---coverage_report_generator=label ---coverage_support=label ---cpu= ---crosstool_top=label ---cs_fdo_absolute_path= ---cs_fdo_instrument= ---cs_fdo_profile=label ---curses={yes,no,auto} ---custom_malloc=label ---cxxopt= ---debug_spawn_scheduler ---nodebug_spawn_scheduler ---define= ---deleted_packages= ---desugar_for_android ---nodesugar_for_android ---desugar_java8_libs ---nodesugar_java8_libs ---device_debug_entitlements ---nodevice_debug_entitlements ---discard_analysis_cache ---nodiscard_analysis_cache ---disk_cache=path ---distdir= ---dynamic_local_execution_delay= ---dynamic_local_strategy= ---dynamic_mode={off,default,fully} ---dynamic_remote_strategy= ---embed_label= ---enable_bzlmod ---noenable_bzlmod ---enable_fdo_profile_absolute_path ---noenable_fdo_profile_absolute_path ---enable_platform_specific_config ---noenable_platform_specific_config ---enable_runfiles={auto,yes,no} ---noenable_runfiles ---enforce_constraints ---noenforce_constraints ---execution_log_binary_file=path ---execution_log_json_file=path ---execution_log_sort ---noexecution_log_sort ---expand_test_suites ---noexpand_test_suites ---experimental_action_listener= ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_add_exec_constraints_to_targets= ---experimental_allow_android_library_deps_without_srcs ---noexperimental_allow_android_library_deps_without_srcs ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_android_compress_java_resources ---noexperimental_android_compress_java_resources ---experimental_android_databinding_v2 ---noexperimental_android_databinding_v2 ---experimental_android_resource_shrinking ---noexperimental_android_resource_shrinking ---experimental_android_rewrite_dexes_with_rex ---noexperimental_android_rewrite_dexes_with_rex ---experimental_android_use_parallel_dex2oat ---noexperimental_android_use_parallel_dex2oat ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cancel_concurrent_tests ---noexperimental_cancel_concurrent_tests ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_check_desugar_deps ---noexperimental_check_desugar_deps ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_local_sandbox_action_metrics ---noexperimental_collect_local_sandbox_action_metrics ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_convenience_symlinks={normal,clean,ignore,log_only} ---experimental_convenience_symlinks_bep_event ---noexperimental_convenience_symlinks_bep_event ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_docker_image= ---experimental_docker_privileged ---noexperimental_docker_privileged ---experimental_docker_use_customized_images ---noexperimental_docker_use_customized_images ---experimental_docker_verbose ---noexperimental_docker_verbose ---experimental_downloader_config= ---experimental_dynamic_exclude_tools ---noexperimental_dynamic_exclude_tools ---experimental_dynamic_ignore_local_signals= ---experimental_dynamic_local_load_factor= ---experimental_dynamic_slow_remote_time= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_docker_sandbox ---noexperimental_enable_docker_sandbox ---experimental_enable_objc_cc_deps ---noexperimental_enable_objc_cc_deps ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_execution_log_file=path ---experimental_execution_log_spawn_metrics ---noexperimental_execution_log_spawn_metrics ---experimental_extra_action_filter= ---experimental_extra_action_top_level_only ---noexperimental_extra_action_top_level_only ---experimental_fetch_all_coverage_outputs ---noexperimental_fetch_all_coverage_outputs ---experimental_filter_library_jar_with_program_jar ---noexperimental_filter_library_jar_with_program_jar ---experimental_gc_thrashing_limits= ---experimental_generate_llvm_lcov ---noexperimental_generate_llvm_lcov ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_import_deps_checking={off,warning,error} ---experimental_include_xcode_execution_requirements ---noexperimental_include_xcode_execution_requirements ---experimental_inmemory_dotd_files ---noexperimental_inmemory_dotd_files ---experimental_inmemory_jdeps_files ---noexperimental_inmemory_jdeps_files ---experimental_inprocess_symlink_creation ---noexperimental_inprocess_symlink_creation ---experimental_j2objc_header_map ---noexperimental_j2objc_header_map ---experimental_j2objc_shorter_header_path ---noexperimental_j2objc_shorter_header_path ---experimental_java_classpath={off,javabuilder,bazel} ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_limit_android_lint_to_android_constrained_java ---noexperimental_limit_android_lint_to_android_constrained_java ---experimental_materialize_param_files_directly ---noexperimental_materialize_param_files_directly ---experimental_multi_cpu= ---experimental_objc_fastbuild_options= ---experimental_objc_include_scanning ---noexperimental_objc_include_scanning ---experimental_omitfp ---noexperimental_omitfp ---experimental_oom_more_eagerly_threshold= ---experimental_platform_in_output_dir ---noexperimental_platform_in_output_dir ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_prefer_mutual_xcode ---noexperimental_prefer_mutual_xcode ---experimental_prioritize_local_actions ---noexperimental_prioritize_local_actions ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_proto_descriptor_sets_include_source_info ---noexperimental_proto_descriptor_sets_include_source_info ---experimental_proto_extra_actions ---noexperimental_proto_extra_actions ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remotable_source_manifests ---noexperimental_remotable_source_manifests ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_eviction_retries= ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_repository_resolved_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_retain_test_configuration_across_testonly ---noexperimental_retain_test_configuration_across_testonly ---experimental_run_android_lint_on_java_rules ---noexperimental_run_android_lint_on_java_rules ---experimental_run_validations ---noexperimental_run_validations ---experimental_sandbox_async_tree_delete_idle_threads= ---experimental_sandbox_memory_limit_mb= ---experimental_sandboxfs_map_symlink_targets ---noexperimental_sandboxfs_map_symlink_targets ---experimental_sandboxfs_path= ---experimental_save_feature_state ---noexperimental_save_feature_state ---experimental_scale_timeouts= ---experimental_shrink_worker_pool ---noexperimental_shrink_worker_pool ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_spawn_scheduler ---experimental_split_coverage_postprocessing ---noexperimental_split_coverage_postprocessing ---experimental_split_xml_generation ---noexperimental_split_xml_generation ---experimental_starlark_cc_import ---noexperimental_starlark_cc_import ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_strict_fileset_output ---noexperimental_strict_fileset_output ---experimental_strict_java_deps={off,warn,error,strict,default} ---experimental_total_worker_memory_limit_mb= ---experimental_ui_max_stdouterr_bytes= ---experimental_unsupported_and_brittle_include_scanning ---noexperimental_unsupported_and_brittle_include_scanning ---experimental_use_hermetic_linux_sandbox ---noexperimental_use_hermetic_linux_sandbox ---experimental_use_llvm_covmap ---noexperimental_use_llvm_covmap ---experimental_use_sandboxfs={auto,yes,no} ---noexperimental_use_sandboxfs ---experimental_use_validation_aspect ---noexperimental_use_validation_aspect ---experimental_use_windows_sandbox={auto,yes,no} ---noexperimental_use_windows_sandbox ---experimental_verify_repository_rules= ---experimental_windows_sandbox_path= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_worker_as_resource ---noexperimental_worker_as_resource ---experimental_worker_cancellation ---noexperimental_worker_cancellation ---experimental_worker_memory_limit_mb= ---experimental_worker_metrics_poll_interval= ---experimental_worker_multiplex ---noexperimental_worker_multiplex ---experimental_worker_multiplex_sandboxing ---noexperimental_worker_multiplex_sandboxing ---experimental_worker_sandbox_hardening ---noexperimental_worker_sandbox_hardening ---experimental_worker_strict_flagfiles ---noexperimental_worker_strict_flagfiles ---experimental_workspace_rules_log_file=path ---explain=path ---explicit_java_test_deps ---noexplicit_java_test_deps ---extra_execution_platforms= ---extra_toolchains= ---fat_apk_cpu= ---fat_apk_hwasan ---nofat_apk_hwasan ---fdo_instrument= ---fdo_optimize= ---fdo_prefetch_hints=label ---fdo_profile=label ---features= ---fission= ---flag_alias= ---flaky_test_attempts= ---force_pic ---noforce_pic ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---genrule_strategy= ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---grte_top=label ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---high_priority_workers= ---host_action_env= ---host_compilation_mode={fastbuild,dbg,opt} ---host_compiler= ---host_conlyopt= ---host_copt= ---host_cpu= ---host_crosstool_top=label ---host_cxxopt= ---host_features= ---host_force_python={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---host_grte_top=label ---host_java_launcher=label ---host_javacopt= ---host_jvmopt= ---host_linkopt= ---host_macos_minimum_os= ---host_per_file_copt= ---host_platform=label ---host_swiftcopt= ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---ignore_unsupported_sandboxing ---noignore_unsupported_sandboxing ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_always_include_files_in_data ---noincompatible_always_include_files_in_data ---incompatible_auto_exec_groups ---noincompatible_auto_exec_groups ---incompatible_avoid_conflict_dlls ---noincompatible_avoid_conflict_dlls ---incompatible_check_sharding_support ---noincompatible_check_sharding_support ---incompatible_check_testonly_for_output_files ---noincompatible_check_testonly_for_output_files ---incompatible_check_visibility_for_toolchains ---noincompatible_check_visibility_for_toolchains ---incompatible_config_setting_private_default_visibility ---noincompatible_config_setting_private_default_visibility ---incompatible_default_to_explicit_init_py ---noincompatible_default_to_explicit_init_py ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_expand_if_all_available_in_flag_set ---noincompatible_disable_expand_if_all_available_in_flag_set ---incompatible_disable_native_android_rules ---noincompatible_disable_native_android_rules ---incompatible_disable_native_apple_binary_rule ---noincompatible_disable_native_apple_binary_rule ---incompatible_disable_runtimes_filegroups ---noincompatible_disable_runtimes_filegroups ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_legacy_py_provider ---noincompatible_disallow_legacy_py_provider ---incompatible_disallow_sdk_frameworks_attributes ---noincompatible_disallow_sdk_frameworks_attributes ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_dont_emit_static_libgcc ---noincompatible_dont_emit_static_libgcc ---incompatible_dont_enable_host_nonhost_crosstool_features ---noincompatible_dont_enable_host_nonhost_crosstool_features ---incompatible_dont_use_javasourceinfoprovider ---noincompatible_dont_use_javasourceinfoprovider ---incompatible_enable_android_toolchain_resolution ---noincompatible_enable_android_toolchain_resolution ---incompatible_enable_apple_toolchain_resolution ---noincompatible_enable_apple_toolchain_resolution ---incompatible_enforce_config_setting_visibility ---noincompatible_enforce_config_setting_visibility ---incompatible_exclusive_test_sandboxed ---noincompatible_exclusive_test_sandboxed ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_force_strict_header_check_from_starlark ---noincompatible_force_strict_header_check_from_starlark ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_legacy_local_fallback ---noincompatible_legacy_local_fallback ---incompatible_linkopts_in_user_link_flags ---noincompatible_linkopts_in_user_link_flags ---incompatible_make_thinlto_command_lines_standalone ---noincompatible_make_thinlto_command_lines_standalone ---incompatible_merge_genfiles_directory ---noincompatible_merge_genfiles_directory ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_objc_alwayslink_by_default ---noincompatible_objc_alwayslink_by_default ---incompatible_objc_linking_info_migration ---noincompatible_objc_linking_info_migration ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_py2_outputs_are_suffixed ---noincompatible_py2_outputs_are_suffixed ---incompatible_py3_is_default ---noincompatible_py3_is_default ---incompatible_python_disable_py2 ---noincompatible_python_disable_py2 ---incompatible_python_disallow_native_rules ---noincompatible_python_disallow_native_rules ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remote_use_new_exit_code_for_lost_inputs ---noincompatible_remote_use_new_exit_code_for_lost_inputs ---incompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---noincompatible_remove_cpu_and_compiler_attributes_from_cc_toolchain ---incompatible_remove_legacy_whole_archive ---noincompatible_remove_legacy_whole_archive ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_ctx_in_configure_features ---noincompatible_require_ctx_in_configure_features ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_sandbox_hermetic_tmp ---noincompatible_sandbox_hermetic_tmp ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_strict_action_env ---noincompatible_strict_action_env ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_use_host_features ---noincompatible_use_host_features ---incompatible_use_platforms_repo_for_constraints ---noincompatible_use_platforms_repo_for_constraints ---incompatible_use_python_toolchains ---noincompatible_use_python_toolchains ---incompatible_validate_top_level_header_inclusions ---noincompatible_validate_top_level_header_inclusions ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---incremental_dexing ---noincremental_dexing ---instrument_test_targets ---noinstrument_test_targets ---instrumentation_filter= ---interface_shared_objects ---nointerface_shared_objects ---internal_spawn_scheduler ---nointernal_spawn_scheduler ---ios_memleaks ---noios_memleaks ---ios_minimum_os= ---ios_multi_cpus= ---ios_sdk_version= ---ios_signing_cert_name= ---ios_simulator_device= ---ios_simulator_version= ---j2objc_translation_flags= ---java_debug ---java_deps ---nojava_deps ---java_header_compilation ---nojava_header_compilation ---java_language_version= ---java_launcher=label ---java_runtime_version= ---javacopt= ---jobs= ---jvmopt= ---keep_going ---nokeep_going ---keep_state_after_build ---nokeep_state_after_build ---legacy_external_runfiles ---nolegacy_external_runfiles ---legacy_important_outputs ---nolegacy_important_outputs ---legacy_main_dex_list_generator=label ---legacy_whole_archive ---nolegacy_whole_archive ---linkopt= ---loading_phase_threads= ---local_cpu_resources= ---local_extra_resources= ---local_ram_resources= ---local_termination_grace_seconds= ---local_test_jobs= ---lockfile_mode={off,update,error} ---logging= ---ltobackendopt= ---ltoindexopt= ---macos_cpus= ---macos_minimum_os= ---macos_sdk_version= ---materialize_param_files ---nomaterialize_param_files ---max_computation_steps= ---max_config_changes_to_show= ---max_test_output_bytes= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---minimum_os_version= ---modify_execution_info= ---nested_set_depth_limit= ---objc_debug_with_GLIBCXX ---noobjc_debug_with_GLIBCXX ---objc_enable_binary_stripping ---noobjc_enable_binary_stripping ---objc_generate_linkmap ---noobjc_generate_linkmap ---objc_use_dotd_pruning ---noobjc_use_dotd_pruning ---objccopt= ---output_filter= ---output_groups= ---override_module= ---override_repository= ---package_path= ---per_file_copt= ---per_file_ltobackendopt= ---persistent_android_dex_desugar ---persistent_android_resource_processor ---persistent_multiplex_android_dex_desugar ---persistent_multiplex_android_resource_processor ---persistent_multiplex_android_tools ---platform_mappings=path ---platform_suffix= ---platforms= ---plugin= ---print_relative_test_log_paths ---noprint_relative_test_log_paths ---process_headers_in_dependencies ---noprocess_headers_in_dependencies ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---progress_report_interval= ---proguard_top=label ---propeller_optimize=label ---propeller_optimize_absolute_cc_profile= ---propeller_optimize_absolute_ld_profile= ---proto_compiler=label ---proto_toolchain_for_cc=label ---proto_toolchain_for_j2objc=label ---proto_toolchain_for_java=label ---proto_toolchain_for_javalite=label ---protocopt= ---python2_path= ---python3_path= ---python_native_rules_allowlist=label ---python_path= ---python_top=label ---python_version={py2,py3,py2and3,py2only,py3only,_internal_sentinel} ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---reuse_sandbox_directories ---noreuse_sandbox_directories ---run_under= ---run_validations ---norun_validations ---runs_per_test= ---runs_per_test_detects_flakes ---noruns_per_test_detects_flakes ---sandbox_add_mount_pair= ---sandbox_base= ---sandbox_block_path= ---sandbox_debug ---nosandbox_debug ---sandbox_default_allow_network ---nosandbox_default_allow_network ---sandbox_explicit_pseudoterminal ---nosandbox_explicit_pseudoterminal ---sandbox_fake_hostname ---nosandbox_fake_hostname ---sandbox_fake_username ---nosandbox_fake_username ---sandbox_tmpfs_path= ---sandbox_writable_path= ---save_temps ---nosave_temps ---share_native_deps ---noshare_native_deps ---shell_executable=path ---show_loading_progress ---noshow_loading_progress ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_result= ---show_timestamps ---noshow_timestamps ---skip_incompatible_explicit_targets ---noskip_incompatible_explicit_targets ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---spawn_strategy= ---stamp ---nostamp ---starlark_cpu_profile= ---strategy= ---strategy_regexp= ---strict_filesets ---nostrict_filesets ---strict_proto_deps={off,warn,error,strict,default} ---strict_public_imports={off,warn,error,strict,default} ---strict_system_includes ---nostrict_system_includes ---strip={always,sometimes,never} ---stripopt= ---subcommands={true,pretty_print,false} ---swiftcopt= ---symlink_prefix= ---target_environment= ---target_pattern_file= ---target_platform_fallback= ---test_arg= ---test_env= ---test_filter= ---test_keep_going ---notest_keep_going ---test_lang_filters= ---test_output={summary,errors,all,streamed} ---test_result_expiration= ---test_runner_fail_fast ---notest_runner_fail_fast ---test_sharding_strategy= ---test_size_filters= ---test_strategy= ---test_summary={short,terse,detailed,none,testcase} ---test_tag_filters= ---test_timeout= ---test_timeout_filters= ---test_tmpdir=path ---test_verbose_timeout_warnings ---notest_verbose_timeout_warnings ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_java_language_version= ---tool_java_runtime_version= ---tool_tag= ---toolchain_resolution_debug= ---track_incremental_state ---notrack_incremental_state ---trim_test_configuration ---notrim_test_configuration ---tvos_cpus= ---tvos_minimum_os= ---tvos_sdk_version= ---tvos_simulator_device= ---tvos_simulator_version= ---ui_actions_shown= ---ui_event_filters= ---use_ijars ---nouse_ijars ---use_singlejar_apkbuilder ---nouse_singlejar_apkbuilder ---use_target_platform_for_tests ---nouse_target_platform_for_tests ---verbose_explanations ---noverbose_explanations ---verbose_failures ---noverbose_failures ---verbose_test_summary ---noverbose_test_summary ---watchfs ---nowatchfs ---watchos_cpus= ---watchos_minimum_os= ---watchos_sdk_version= ---watchos_simulator_device= ---watchos_simulator_version= ---worker_extra_flag= ---worker_max_instances= ---worker_max_multiplex_instances= ---worker_quit_after_build ---noworker_quit_after_build ---worker_sandboxing ---noworker_sandboxing ---worker_verbose ---noworker_verbose ---workspace_status_command=path ---xbinary_fdo=label ---xcode_version= ---xcode_version_config=label ---zip_undeclared_test_outputs ---nozip_undeclared_test_outputs -" -BAZEL_COMMAND_VERSION_FLAGS=" ---allow_yanked_versions= ---announce_rc ---noannounce_rc ---attempt_to_print_relative_paths ---noattempt_to_print_relative_paths ---bep_maximum_open_remote_upload_files= ---bes_backend= ---bes_check_preceding_lifecycle_events ---nobes_check_preceding_lifecycle_events ---bes_header= ---bes_instance_name= ---bes_keywords= ---bes_lifecycle_events ---nobes_lifecycle_events ---bes_oom_finish_upload_timeout= ---bes_outerr_buffer_size= ---bes_outerr_chunk_size= ---bes_proxy= ---bes_results_url= ---bes_system_keywords= ---bes_timeout= ---bes_upload_mode={wait_for_upload_complete,nowait_for_upload_complete,fully_async} ---build_event_binary_file= ---build_event_binary_file_path_conversion ---nobuild_event_binary_file_path_conversion ---build_event_json_file= ---build_event_json_file_path_conversion ---nobuild_event_json_file_path_conversion ---build_event_max_named_set_of_file_entries= ---build_event_publish_all_actions ---nobuild_event_publish_all_actions ---build_event_text_file= ---build_event_text_file_path_conversion ---nobuild_event_text_file_path_conversion ---build_metadata= ---check_bazel_compatibility={error,warning,off} ---check_bzl_visibility ---nocheck_bzl_visibility ---check_direct_dependencies={off,warning,error} ---color={yes,no,auto} ---config= ---curses={yes,no,auto} ---disk_cache=path ---distdir= ---enable_bzlmod ---noenable_bzlmod ---enable_platform_specific_config ---noenable_platform_specific_config ---experimental_action_resource_set ---noexperimental_action_resource_set ---experimental_allow_tags_propagation ---noexperimental_allow_tags_propagation ---experimental_allow_top_level_aspects_parameters ---noexperimental_allow_top_level_aspects_parameters ---experimental_analysis_test_call ---noexperimental_analysis_test_call ---experimental_announce_profile_path ---noexperimental_announce_profile_path ---experimental_bep_target_summary ---noexperimental_bep_target_summary ---experimental_build_event_expand_filesets ---noexperimental_build_event_expand_filesets ---experimental_build_event_fully_resolve_fileset_symlinks ---noexperimental_build_event_fully_resolve_fileset_symlinks ---experimental_build_event_upload_max_retries= ---experimental_build_event_upload_retry_minimum_delay= ---experimental_build_event_upload_strategy= ---experimental_bzl_visibility ---noexperimental_bzl_visibility ---experimental_cc_shared_library ---noexperimental_cc_shared_library ---experimental_collect_load_average_in_profiler ---noexperimental_collect_load_average_in_profiler ---experimental_collect_pressure_stall_indicators ---noexperimental_collect_pressure_stall_indicators ---experimental_collect_resource_estimation ---noexperimental_collect_resource_estimation ---experimental_collect_system_network_usage ---noexperimental_collect_system_network_usage ---experimental_collect_worker_data_in_profiler ---noexperimental_collect_worker_data_in_profiler ---experimental_command_profile ---noexperimental_command_profile ---experimental_credential_helper= ---experimental_credential_helper_cache_duration= ---experimental_credential_helper_timeout= ---experimental_disable_external_package ---noexperimental_disable_external_package ---experimental_downloader_config= ---experimental_enable_android_migration_apis ---noexperimental_enable_android_migration_apis ---experimental_enable_scl_dialect ---noexperimental_enable_scl_dialect ---experimental_gc_thrashing_limits= ---experimental_get_fixed_configured_action_env ---noexperimental_get_fixed_configured_action_env ---experimental_google_legacy_api ---noexperimental_google_legacy_api ---experimental_guard_against_concurrent_changes ---noexperimental_guard_against_concurrent_changes ---experimental_java_library_export ---noexperimental_java_library_export ---experimental_lazy_template_expansion ---noexperimental_lazy_template_expansion ---experimental_oom_more_eagerly_threshold= ---experimental_platforms_api ---noexperimental_platforms_api ---experimental_profile_additional_tasks= ---experimental_profile_include_primary_output ---noexperimental_profile_include_primary_output ---experimental_profile_include_target_label ---noexperimental_profile_include_target_label ---experimental_record_metrics_for_all_mnemonics ---noexperimental_record_metrics_for_all_mnemonics ---experimental_remote_cache_async ---noexperimental_remote_cache_async ---experimental_remote_cache_ttl= ---experimental_remote_capture_corrupted_outputs=path ---experimental_remote_discard_merkle_trees ---noexperimental_remote_discard_merkle_trees ---experimental_remote_downloader= ---experimental_remote_downloader_local_fallback ---noexperimental_remote_downloader_local_fallback ---experimental_remote_execution_keepalive ---noexperimental_remote_execution_keepalive ---experimental_remote_mark_tool_inputs ---noexperimental_remote_mark_tool_inputs ---experimental_remote_merkle_tree_cache ---noexperimental_remote_merkle_tree_cache ---experimental_remote_merkle_tree_cache_size= ---experimental_repo_remote_exec ---noexperimental_repo_remote_exec ---experimental_repository_cache_hardlinks ---noexperimental_repository_cache_hardlinks ---experimental_repository_cache_urls_as_default_canonical_id ---noexperimental_repository_cache_urls_as_default_canonical_id ---experimental_repository_downloader_retries= ---experimental_repository_hash_file= ---experimental_resolved_file_instead_of_workspace= ---experimental_scale_timeouts= ---experimental_sibling_repository_layout ---noexperimental_sibling_repository_layout ---experimental_skymeld_ui ---noexperimental_skymeld_ui ---experimental_stream_log_file_uploads ---noexperimental_stream_log_file_uploads ---experimental_ui_max_stdouterr_bytes= ---experimental_verify_repository_rules= ---experimental_windows_watchfs ---noexperimental_windows_watchfs ---experimental_workspace_rules_log_file=path ---gc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---nogc_thrashing_limits_retained_heap_limiter_mutually_exclusive ---generate_json_trace_profile={auto,yes,no} ---nogenerate_json_trace_profile ---gnu_format ---nognu_format ---google_auth_scopes= ---google_credentials= ---google_default_credentials ---nogoogle_default_credentials ---grpc_keepalive_time= ---grpc_keepalive_timeout= ---heap_dump_on_oom ---noheap_dump_on_oom ---heuristically_drop_nodes ---noheuristically_drop_nodes ---http_connector_attempts= ---http_connector_retry_max_timeout= ---http_timeout_scaling= ---ignore_dev_dependency ---noignore_dev_dependency ---incompatible_always_check_depset_elements ---noincompatible_always_check_depset_elements ---incompatible_depset_for_libraries_to_link_getter ---noincompatible_depset_for_libraries_to_link_getter ---incompatible_disable_starlark_host_transitions ---noincompatible_disable_starlark_host_transitions ---incompatible_disable_target_provider_fields ---noincompatible_disable_target_provider_fields ---incompatible_disallow_empty_glob ---noincompatible_disallow_empty_glob ---incompatible_disallow_legacy_javainfo ---noincompatible_disallow_legacy_javainfo ---incompatible_disallow_struct_provider_syntax ---noincompatible_disallow_struct_provider_syntax ---incompatible_disallow_symlink_file_to_dir ---noincompatible_disallow_symlink_file_to_dir ---incompatible_do_not_split_linking_cmdline ---noincompatible_do_not_split_linking_cmdline ---incompatible_existing_rules_immutable_view ---noincompatible_existing_rules_immutable_view ---incompatible_fix_package_group_reporoot_syntax ---noincompatible_fix_package_group_reporoot_syntax ---incompatible_java_common_parameters ---noincompatible_java_common_parameters ---incompatible_new_actions_api ---noincompatible_new_actions_api ---incompatible_no_attr_license ---noincompatible_no_attr_license ---incompatible_no_implicit_file_export ---noincompatible_no_implicit_file_export ---incompatible_no_rule_outputs_param ---noincompatible_no_rule_outputs_param ---incompatible_package_group_has_public_syntax ---noincompatible_package_group_has_public_syntax ---incompatible_remote_build_event_upload_respect_no_cache ---noincompatible_remote_build_event_upload_respect_no_cache ---incompatible_remote_dangling_symlinks ---noincompatible_remote_dangling_symlinks ---incompatible_remote_disallow_symlink_in_tree_artifact ---noincompatible_remote_disallow_symlink_in_tree_artifact ---incompatible_remote_downloader_send_all_headers ---noincompatible_remote_downloader_send_all_headers ---incompatible_remote_output_paths_relative_to_input_root ---noincompatible_remote_output_paths_relative_to_input_root ---incompatible_remote_results_ignore_disk ---noincompatible_remote_results_ignore_disk ---incompatible_remote_symlinks ---noincompatible_remote_symlinks ---incompatible_remove_rule_name_parameter ---noincompatible_remove_rule_name_parameter ---incompatible_require_linker_input_cc_api ---noincompatible_require_linker_input_cc_api ---incompatible_run_shell_command_string ---noincompatible_run_shell_command_string ---incompatible_stop_exporting_language_modules ---noincompatible_stop_exporting_language_modules ---incompatible_struct_has_no_methods ---noincompatible_struct_has_no_methods ---incompatible_top_level_aspects_require_providers ---noincompatible_top_level_aspects_require_providers ---incompatible_unambiguous_label_stringification ---noincompatible_unambiguous_label_stringification ---incompatible_use_cc_configure_from_rules_cc ---noincompatible_use_cc_configure_from_rules_cc ---incompatible_visibility_private_attributes_at_definition ---noincompatible_visibility_private_attributes_at_definition ---keep_state_after_build ---nokeep_state_after_build ---legacy_important_outputs ---nolegacy_important_outputs ---lockfile_mode={off,update,error} ---logging= ---max_computation_steps= ---memory_profile=path ---memory_profile_stable_heap_parameters= ---nested_set_depth_limit= ---override_module= ---override_repository= ---profile=path ---progress_in_terminal_title ---noprogress_in_terminal_title ---record_full_profiler_data ---norecord_full_profiler_data ---registry= ---remote_accept_cached ---noremote_accept_cached ---remote_build_event_upload={all,minimal} ---remote_bytestream_uri_prefix= ---remote_cache= ---remote_cache_compression ---noremote_cache_compression ---remote_cache_header= ---remote_default_exec_properties= ---remote_default_platform_properties= ---remote_download_minimal ---remote_download_outputs={all,minimal,toplevel} ---remote_download_symlink_template= ---remote_download_toplevel ---remote_downloader_header= ---remote_exec_header= ---remote_execution_priority= ---remote_executor= ---remote_grpc_log=path ---remote_header= ---remote_instance_name= ---remote_local_fallback ---noremote_local_fallback ---remote_local_fallback_strategy= ---remote_max_connections= ---remote_print_execution_messages={failure,success,all} ---remote_proxy= ---remote_result_cache_priority= ---remote_retries= ---remote_retry_max_delay= ---remote_timeout= ---remote_upload_local_results ---noremote_upload_local_results ---remote_verify_downloads ---noremote_verify_downloads ---repo_env= ---repository_cache=path ---repository_disable_download ---norepository_disable_download ---show_progress ---noshow_progress ---show_progress_rate_limit= ---show_timestamps ---noshow_timestamps ---skyframe_high_water_mark_full_gc_drops_per_invocation= ---skyframe_high_water_mark_minor_gc_drops_per_invocation= ---skyframe_high_water_mark_threshold= ---slim_profile ---noslim_profile ---starlark_cpu_profile= ---tls_certificate= ---tls_client_certificate= ---tls_client_key= ---tool_tag= ---track_incremental_state ---notrack_incremental_state ---ui_actions_shown= ---ui_event_filters= ---watchfs ---nowatchfs -" diff --git a/bazel/bazelrc/bazel6.bazelrc b/bazel/bazelrc/bazel7.bazelrc similarity index 77% rename from bazel/bazelrc/bazel6.bazelrc rename to bazel/bazelrc/bazel7.bazelrc index 11a1c67fa..212c26e7d 100644 --- a/bazel/bazelrc/bazel6.bazelrc +++ b/bazel/bazelrc/bazel7.bazelrc @@ -9,7 +9,3 @@ query --noexperimental_check_external_repository_files # build. # Docs: https://bazel.build/reference/command-line-reference#flag--reuse_sandbox_directories build --reuse_sandbox_directories - -# Avoid this flag being enabled by remote_download_minimal or remote_download_toplevel -# See https://meroton.com/blog/bazel-6-errors-build-without-the-bytes/ -build --noexperimental_action_cache_store_output_metadata diff --git a/bazel/bazelrc/ci.bazelrc b/bazel/bazelrc/ci.bazelrc index 4bc188e3d..dbae2df5e 100644 --- a/bazel/bazelrc/ci.bazelrc +++ b/bazel/bazelrc/ci.bazelrc @@ -62,6 +62,10 @@ build --remote_local_fallback # Docs: https://bazel.build/reference/command-line-reference#flag--grpc_keepalive_time build --grpc_keepalive_time=30s +# Use fallbacks in case proxy.golang.org is not reachable. +# Docs: https://go.dev/ref/mod#goproxy-protocol +common '--repo_env=GOPROXY=https://proxy.golang.org|https://goproxy.io|direct' + ###################################### # Edgeless specific # diff --git a/bazel/bazelrc/correctness.bazelrc b/bazel/bazelrc/correctness.bazelrc index 6c7934609..61a469071 100644 --- a/bazel/bazelrc/correctness.bazelrc +++ b/bazel/bazelrc/correctness.bazelrc @@ -44,3 +44,16 @@ query --experimental_allow_tags_propagation # https://github.com/bazelbuild/bazel/issues/10076. # Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_default_to_explicit_init_py build --incompatible_default_to_explicit_init_py + +# Set default value of `allow_empty` to `False` in `glob()`. This prevents a common mistake when +# attempting to use `glob()` to match files in a subdirectory that is opaque to the current package +# because it contains a BUILD file. See https://github.com/bazelbuild/bazel/issues/8195. +# Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_disallow_empty_glob +common --incompatible_disallow_empty_glob + +# Always download coverage files for tests from the remote cache. By default, coverage files are not +# downloaded on test result cache hits when --remote_download_minimal is enabled, making it impossible +# to generate a full coverage report. +# Docs: https://bazel.build/reference/command-line-reference#flag--experimental_fetch_all_coverage_outputs +# detching remote cache results +test --experimental_fetch_all_coverage_outputs diff --git a/bazel/ci/BUILD.bazel b/bazel/ci/BUILD.bazel index 2570e53c0..3fdcdcc8b 100644 --- a/bazel/ci/BUILD.bazel +++ b/bazel/ci/BUILD.bazel @@ -1,6 +1,7 @@ -load("@bazel_gazelle//:def.bzl", "gazelle") -load("@com_github_ash2k_bazel_tools//multirun:def.bzl", "command", "multirun") -load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier", "buildifier_test") +load("@buildifier_prebuilt//:rules.bzl", "buildifier", "buildifier_test") +load("@com_github_ash2k_bazel_tools//multirun:def.bzl", "multirun") +load("@gazelle//:def.bzl", "gazelle") +load("//bazel/ci:go_bin_for_host.bzl", "go_bin_for_host") load("//bazel/ci:proto_targets.bzl", "proto_targets") load("//bazel/sh:def.bzl", "noop_warn", "repo_command", "sh_template") @@ -9,18 +10,6 @@ required_tags = [ "integration", ] -# TODO(malt3): Remove this once we have a better solution for -# gazelle not respecting the default go env. -command( - name = "cmd_gazelle_update_repos", - command = ":gazelle_update_repos", - environment = { - "GOPROXY": "https://proxy.golang.org,direct", - "GOSUMDB": "sum.golang.org", - "GOTOOLCHAIN": "local", - }, -) - gazelle( name = "gazelle_generate", build_tags = required_tags, @@ -33,18 +22,6 @@ gazelle( mode = "diff", ) -gazelle( - name = "gazelle_update_repos", - args = [ - "-from_file=go.work", - "-to_macro=bazel/toolchains/go_module_deps.bzl%go_dependencies", - "-build_file_proto_mode=disable_global", - "-build_file_generation=on", - "-prune", - ], - command = "update-repos", -) - buildifier_test( name = "buildifier_check", timeout = "short", @@ -54,7 +31,7 @@ buildifier_test( no_sandbox = True, tags = ["no-remote-exec"], verbose = True, - workspace = "//:WORKSPACE.bazel", + workspace = "//:WORKSPACE.bzlmod", ) buildifier( @@ -67,12 +44,23 @@ buildifier( ) sh_template( - name = "go_mod_tidy", + name = "keep_sorted", data = [ - "@go_sdk//:bin/go", + "@com_github_google_keep_sorted//:keep-sorted", ], substitutions = { - "@@GO@@": "$(rootpath @go_sdk//:bin/go)", + "@@KEEP_SORTED@@": "$(rootpath @com_github_google_keep_sorted//:keep-sorted)", + }, + template = "keep_sorted.sh.in", +) + +sh_template( + name = "go_mod_tidy", + data = [ + ":go_bin_for_host", + ], + substitutions = { + "@@GO@@": "$(rootpath :go_bin_for_host)", }, template = "go_tidy.sh.in", ) @@ -246,10 +234,10 @@ sh_template( name = "golangci_lint", data = [ ":com_github_golangci_golangci_lint", - "@go_sdk//:bin/go", + ":go_bin_for_host", ], substitutions = { - "@@GO@@": "$(rootpath @go_sdk//:bin/go)", + "@@GO@@": "$(rootpath :go_bin_for_host)", "@@GOLANGCI-LINT@@": "$(rootpath :com_github_golangci_golangci_lint)", }, template = "golangci_lint.sh.in", @@ -279,11 +267,11 @@ sh_template( sh_template( name = "golicenses_check", data = [ + ":go_bin_for_host", "@com_github_google_go_licenses//:go-licenses", - "@go_sdk//:bin/go", ], substitutions = { - "@@GO@@": "$(rootpath @go_sdk//:bin/go)", + "@@GO@@": "$(rootpath :go_bin_for_host)", "@@GO_LICENSES@@": "$(rootpath @com_github_google_go_licenses//:go-licenses)", }, template = "golicenses.sh.in", @@ -299,12 +287,14 @@ sh_template( sh_template( name = "govulncheck", data = [ - "@go_sdk//:bin/go", + ":go_bin_for_host", + "@jq_toolchains//:resolved_toolchain", "@org_golang_x_vuln//cmd/govulncheck", ], substitutions = { - "@@GO@@": "$(rootpath @go_sdk//:bin/go)", + "@@GO@@": "$(rootpath :go_bin_for_host)", "@@GOVULNCHECK@@": "$(rootpath @org_golang_x_vuln//cmd/govulncheck:govulncheck)", + "@@JQ@@": "$(rootpath @jq_toolchains//:resolved_toolchain)", }, template = "govulncheck.sh.in", ) @@ -334,15 +324,15 @@ sh_template( data = [ ":com_github_helm_helm", ":com_github_siderolabs_talos_hack_docgen", + ":go_bin_for_host", "//internal/attestation/measurements/measurement-generator", "//internal/versions/hash-generator", - "@go_sdk//:bin/go", "@org_golang_x_tools//cmd/stringer", "@yq_toolchains//:resolved_toolchain", ], substitutions = { "@@DOCGEN@@": "$(rootpath :com_github_siderolabs_talos_hack_docgen)", - "@@GO@@": "$(rootpath @go_sdk//:bin/go)", + "@@GO@@": "$(rootpath :go_bin_for_host)", "@@HASH_GENERATOR@@": "$(rootpath //internal/versions/hash-generator:hash-generator)", "@@HELM@@": "$(rootpath :com_github_helm_helm)", "@@MEASUREMENT_GENERATOR@@": "$(rootpath //internal/attestation/measurements/measurement-generator:measurement-generator)", @@ -481,13 +471,6 @@ repo_command( command = ":com_github_katexochen_ghh", ) -sh_template( - name = "bazel_container", - data = [], - substitutions = {}, - template = "bazel_container.sh.in", -) - sh_template( name = "unused_gh_actions", data = [], @@ -495,14 +478,19 @@ sh_template( template = "unused_gh_actions.sh.in", ) +go_bin_for_host( + name = "go_bin_for_host", + visibility = ["//visibility:private"], +) + sh_template( name = "gocoverage_diff", data = [ + ":go_bin_for_host", "//hack/gocoverage", - "@go_sdk//:bin/go", ], substitutions = { - "@@GO@@": "$(rootpath @go_sdk//:bin/go)", + "@@GO@@": "$(rootpath :go_bin_for_host)", "@@GOCOVERAGE@@": "$(rootpath //hack/gocoverage:gocoverage)", }, template = "gocoverage_diff.sh.in", @@ -514,29 +502,26 @@ multirun( ":shfmt", ":gofumpt", ":go_mod_tidy", - ":cmd_gazelle_update_repos", ":gazelle_generate", ":buildifier_fix", ":terraform_fmt", ":buf_fmt", ":deps_mirror_fix", - ":bazel_container", + ":keep_sorted", ], jobs = 1, # execute sequentially visibility = ["//visibility:public"], ) multirun( - name = "check", + name = "parallel_checks", testonly = True, commands = [ ":gazelle_check", ":buildifier_check", - ":golangci_lint", ":terraform_check", ":golicenses_check", ":license_header_check", - ":govulncheck", ":deps_mirror_check", ":proto_targets_check", ":unused_gh_actions", @@ -556,16 +541,48 @@ multirun( ) multirun( - name = "generate", + name = "check", + testonly = True, + commands = [ + ":parallel_checks", + ":golangci_lint", + ":govulncheck", + ], + jobs = 1, # execute sequentially to avoid running into memory issues on our CI runners + stop_on_error = False, + visibility = ["//visibility:public"], +) + +multirun( + name = "generate_files", commands = [ ":terraform_gen", "//3rdparty/bazel/com_github_medik8s_node_maintainance_operator:pull_files", + "//3rdparty/bazel/com_github_kubernetes_sigs_aws_load_balancer_controller:pull_files", ":go_generate", ":proto_generate", - ":cli_docgen", - ":terraform_docgen", - ":version_info_gen", ], jobs = 0, # execute concurrently visibility = ["//visibility:public"], ) + +multirun( + name = "generate_docs", + commands = [ + ":cli_docgen", + ":terraform_docgen", + ], + jobs = 0, # execute concurrently + visibility = ["//visibility:public"], +) + +multirun( + name = "generate", + commands = [ + ":generate_files", + ":generate_docs", + ":version_info_gen", + ], + jobs = 1, # execute sequentially + visibility = ["//visibility:public"], +) diff --git a/bazel/ci/bazel_container.sh.in b/bazel/ci/bazel_container.sh.in deleted file mode 100755 index b3539e64f..000000000 --- a/bazel/ci/bazel_container.sh.in +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -###### script header ###### - -lib=$(realpath @@BASE_LIB@@) || exit 1 -stat "${lib}" >> /dev/null || exit 1 - -# shellcheck source=../sh/lib.bash -if ! source "${lib}"; then - echo "Error: could not find import" - exit 1 -fi - -cd "${BUILD_WORKSPACE_DIRECTORY}" - -###### script body ###### - -bazelVer=$(cat .bazelversion) # has no v prefix - -bazelVerContainerfileRegex='(ARG BAZEL_VERSION=)([0-9]+\.[0-9]+\.[0-9]+)' -if [[ ! "$(cat bazel/container/Containerfile)" =~ ${bazelVerContainerfileRegex} ]]; then - echo "Error: no match found in Containerfile" - exit 1 -fi - -bazelVerContainerfile="${BASH_REMATCH[2]}" - -if [[ ${bazelVer} != "${bazelVerContainerfile}" ]]; then - sed -r -i "s/${bazelVerContainerfileRegex}/\\1${bazelVer}/" bazel/container/Containerfile - echo "Containerfile updated, was previously at ${bazelVerContainerfile}" -fi - -bazelVerScriptRegex='(containerImage="ghcr.io/edgelesssys/bazel-container:v)([0-9]+\.[0-9]+\.[0-9]+)' -if [[ ! "$(cat bazel/container/container.sh)" =~ ${bazelVerScriptRegex} ]]; then - echo "Error: no match found in container.sh" - exit 1 -fi - -bazelVerScript="${BASH_REMATCH[2]}" - -if [[ ${bazelVer} != "${bazelVerScript}" ]]; then - # bazelVerScriptRegex contains slashes, so use % as delimiter - sed -r -i "s%${bazelVerScriptRegex}%\\1${bazelVer}%" bazel/container/container.sh - echo "container.sh updated, was previously at ${bazelVerScript}" -fi diff --git a/bazel/ci/go_bin_for_host.bzl b/bazel/ci/go_bin_for_host.bzl new file mode 100644 index 000000000..29721b2e2 --- /dev/null +++ b/bazel/ci/go_bin_for_host.bzl @@ -0,0 +1,29 @@ +""" +Go toolchain for the host platformS +Inspired by https://github.com/bazel-contrib/rules_go/blob/6e4fdcfeb1a333b54ab39ae3413d4ded46d8958d/go/private/rules/go_bin_for_host.bzl +""" + +load("@local_config_platform//:constraints.bzl", "HOST_CONSTRAINTS") + +GO_TOOLCHAIN = "@io_bazel_rules_go//go:toolchain" + +def _ensure_target_cfg(ctx): + if "-exec" in ctx.bin_dir.path or "/host/" in ctx.bin_dir.path: + fail("exec not found") + +def _go_bin_for_host_impl(ctx): + _ensure_target_cfg(ctx) + sdk = ctx.toolchains[GO_TOOLCHAIN].sdk + sdk_files = ctx.runfiles([sdk.go] + sdk.headers.to_list() + sdk.libs.to_list() + sdk.srcs.to_list() + sdk.tools.to_list()) + return [ + DefaultInfo( + files = depset([sdk.go]), + runfiles = sdk_files, + ), + ] + +go_bin_for_host = rule( + implementation = _go_bin_for_host_impl, + toolchains = [GO_TOOLCHAIN], + exec_compatible_with = HOST_CONSTRAINTS, +) diff --git a/bazel/ci/golicenses.sh.in b/bazel/ci/golicenses.sh.in index 3d342d78b..4f3eb78e9 100644 --- a/bazel/ci/golicenses.sh.in +++ b/bazel/ci/golicenses.sh.in @@ -57,9 +57,7 @@ license_report() { AGPL-3.0) case ${pkg} in - github.com/edgelesssys/constellation/v2) ;; - - github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1) ;; + github.com/edgelesssys/go-tdx-qpl) ;; *) not_allowed @@ -69,7 +67,7 @@ license_report() { Unknown) case ${pkg} in - github.com/edgelesssys/go-tdx-qpl/*) ;; + github.com/edgelesssys/constellation/v2/*) ;; *) not_allowed diff --git a/bazel/ci/govulncheck.sh.in b/bazel/ci/govulncheck.sh.in index 0ea07b10c..28da4441f 100644 --- a/bazel/ci/govulncheck.sh.in +++ b/bazel/ci/govulncheck.sh.in @@ -15,6 +15,8 @@ go=$(realpath @@GO@@) stat "${go}" >> /dev/null govulncheck=$(realpath @@GOVULNCHECK@@) stat "${govulncheck}" >> /dev/null +jq=$(realpath @@JQ@@) +stat "${jq}" >> /dev/null cd "${BUILD_WORKSPACE_DIRECTORY}" @@ -24,19 +26,32 @@ submodules=$(${go} list -f '{{.Dir}}' -m) PATH=$(dirname "${go}"):${PATH} +check_module() { + excluded_osvs=( + "GO-2025-3521" # Kubernetes GitRepo Volume Inadvertent Local Repository Access + "GO-2025-3547" # Kubernetes kube-apiserver Vulnerable to Race Condition + "GO-2025-3770" # Host Header Injection which Leads to Open Redirect in RedirectSlashes in github.com/go-chi/chi + ) + + # shellcheck disable=SC2016 # The $ sign in the single quoted string is correct. + CGO_ENABLED=0 ${govulncheck} -C "$1" -format json "./..." | + "${jq}" --argjson excluded "$(printf '"%s"\n' "${excluded_osvs[@]}" | jq -s)" -sr ' + (map(select(.osv) | {"key": .osv.id, "value": .osv.summary}) | from_entries) as $osvs | + map(select( .finding and all($excluded[] != .finding.osv; .) ) | .finding | select( .trace[-1].module | startswith("github.com/edgelesssys/") )) | + group_by(.osv) | + map( {"osv": .[0].osv, "summary": $osvs[.[0].osv], "traces": [.[] | [.trace[] | .module]]} ) | + if length > 0 then halt_error(1) else .[] end' + +} + check() { err=0 - echo "Scanning Go vulnerability DB for knwon vulnerabilities in modules:" + echo "Scanning Go vulnerability DB for known vulnerabilities in modules:" for mod in ${submodules}; do echo " ${mod}" echo -n " " - CGO_ENABLED=0 ${govulncheck} -C "${mod}" "./..." | - tail -n 2 | # Providing some nice output... - tr '\n' ' ' | - sed s/" your code and"// && - printf "\n" || - err=$? + check_module "${mod}" done exit "${err}" diff --git a/bazel/ci/keep_sorted.sh.in b/bazel/ci/keep_sorted.sh.in new file mode 100644 index 000000000..471cf157a --- /dev/null +++ b/bazel/ci/keep_sorted.sh.in @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +###### script header ###### + +lib=$(realpath @@BASE_LIB@@) || exit 1 +stat "${lib}" >> /dev/null || exit 1 + +# shellcheck source=../sh/lib.bash +if ! source "${lib}"; then + echo "Error: could not find import" + exit 1 +fi + +keep_sorted=$(realpath @@KEEP_SORTED@@) +stat "${keep_sorted}" >> /dev/null + +cd "${BUILD_WORKSPACE_DIRECTORY}" + +###### script body ###### + +find . -not -path "./.git/*" -type f | sort | xargs "${keep_sorted}" --mode fix diff --git a/bazel/ci/license_header.sh.in b/bazel/ci/license_header.sh.in index 5dba4e4a4..8278769cd 100644 --- a/bazel/ci/license_header.sh.in +++ b/bazel/ci/license_header.sh.in @@ -25,8 +25,8 @@ noHeader=$( --include='*.go' \ --exclude-dir 3rdparty \ --exclude-dir build \ - -e'SPDX-License-Identifier: AGPL-3.0-only' \ - -e'DO NOT EDIT' + -e'SPDX-License-Identifier: BUSL-1.1' \ + -e'DO NOT EDIT' | { grep -v internal/cloud/openstack/clouds || true; } ) if [[ -z ${noHeader} ]]; then diff --git a/bazel/ci/shellcheck.sh.in b/bazel/ci/shellcheck.sh.in index e68e82a32..cbb1f84a9 100644 --- a/bazel/ci/shellcheck.sh.in +++ b/bazel/ci/shellcheck.sh.in @@ -25,15 +25,14 @@ readarray -t <<< "${scriptsStr}" scripts=("${MAPFILE[@]}") excludeDirs=( + ".direnv" "internal/constellation/helm/charts/cilium" "build" "docs/node_modules" "terraform-provider-constellation/examples" ) -excludeFiles=( - "bazel/bazel-complete.bash" -) +excludeFiles=() echo "The following scripts are excluded by their directory and won't be linted with shellcheck:" for exclude in "${excludeDirs[@]}"; do diff --git a/bazel/ci/terraform.sh.in b/bazel/ci/terraform.sh.in index 121b2313d..777049106 100644 --- a/bazel/ci/terraform.sh.in +++ b/bazel/ci/terraform.sh.in @@ -46,7 +46,6 @@ excludeDirs=( excludeLockDirs=( "build" "terraform-provider-constellation" - "terraform/legacy-module" ) excludeCheckDirs=( @@ -143,6 +142,7 @@ check() { done echo "This may take 5-10 min..." for module in "${terraformLockModules[@]}"; do + echo "Generating lock file for ${module}" ${terraform} -chdir="${module}" init > /dev/null ${terraform} -chdir="${module}" providers lock -platform=linux_arm64 > /dev/null ${terraform} -chdir="${module}" providers lock -platform=linux_amd64 > /dev/null diff --git a/bazel/ci/unused_gh_actions.sh.in b/bazel/ci/unused_gh_actions.sh.in index a4f2a5fd3..e6a5f192f 100755 --- a/bazel/ci/unused_gh_actions.sh.in +++ b/bazel/ci/unused_gh_actions.sh.in @@ -50,7 +50,7 @@ for action in ${actionNames}; do fi done - if ! ${used}; then + if [[ ${used} == "false" && ${action} != ".github/actions/artifact_download" ]]; then echo "Action ${action} is unused" exitcode=1 fi diff --git a/bazel/container/Containerfile b/bazel/container/Containerfile deleted file mode 100644 index 040555e17..000000000 --- a/bazel/container/Containerfile +++ /dev/null @@ -1,36 +0,0 @@ -# syntax=docker/dockerfile:1.5-labs -FROM fedora:38 - -ARG TARGETOS -ARG TARGETARCH -ARG BAZEL_VERSION=6.4.0 -ARG BAZELISK_VERSION=v1.16.0 -ARG BAZELISK_SHA256=168851e70cf5f95c0e215e7f3aaca5132ffc3c8dd8f585a4157b0be2b53cfe32 - -ADD --checksum=sha256:${BAZELISK_SHA256} \ - https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${TARGETOS}-${TARGETARCH} \ - /usr/local/bin/bazelisk - -RUN chmod +x /usr/local/bin/bazelisk && \ - ln -s /usr/local/bin/bazelisk /usr/local/bin/bazel && \ - dnf install -y \ - git \ - diffutils \ - libxcrypt-compat \ - python3 \ - && \ - dnf clean all && \ - groupadd --gid 1000 builder && \ - useradd -rm -d /home/builder -s /bin/bash -g root -u 1000 --gid builder builder && \ - mkdir -p /home/builder/.cache && \ - mkdir -p /workspace && \ - chown -R builder:builder /home/builder/.cache /workspace && \ - git config --global --add safe.directory /workspace - -USER builder -WORKDIR /workspace - -RUN git config --global --add safe.directory /workspace && \ - USE_BAZEL_VERSION=${BAZEL_VERSION} bazel version - -ENTRYPOINT [ "/usr/local/bin/bazel" ] diff --git a/bazel/container/README.md b/bazel/container/README.md deleted file mode 100644 index b3c5079ca..000000000 --- a/bazel/container/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Bazel build container - -This container enables running Bazel inside a container, with the host cache mounted. - -To use the container, run - -```shell -source container.sh -startBazelServer -``` - -You can then execute Bazel commands like you normally would do, as the sourced `bazel` -function shadows binaries you might have in your path: - -```shell -bazel query //... -``` - -To terminate the container, which is running as daemon in the background, execute - -```shell -stopBazelServer -``` diff --git a/bazel/container/container.sh b/bazel/container/container.sh deleted file mode 100644 index 9065e8792..000000000 --- a/bazel/container/container.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -function setup { - (stopBazelServer && sleep 1) || true - - # Ensure that the cache directories exist, so they are not created by docker with root permissions. - mkdir -p "${HOME}/.cache/bazel" - mkdir -p "${HOME}/.cache/shared_bazel_repository_cache" - mkdir -p "${HOME}/.cache/shared_bazel_action_cache" - - if [[ ! -f "${HOME}/.docker/config.json" ]]; then - echo "ERROR: ${HOME}/.docker/config.json does not exist." - echo "Please login into your container registry to create it." - echo "echo | docker login ghcr.io -u --password-stdin" - exit 1 - fi -} - -function startBazelServer { - local containerImage="ghcr.io/edgelesssys/bazel-container:v6.4.0" - local containerName="bazeld" - - setup - - # In-container .bazelrc overwrite. - mkdir -p "/tmp/bazel-container" - cat << EOF > "/tmp/bazel-container/.bazelrc" -startup --output_user_root=/home/${USER}/.cache/bazel/_bazel_${USER} -EOF - - local hostWorkspaceDir - hostWorkspaceDir="$(git rev-parse --show-toplevel)" - if [[ $? -ne 0 ]]; then - echo Could not find git repository root. Are you in a git repository? - return 1 - fi - - echo Starting bazel container as daemon... - echo You can stop this command using: - echo docker kill "${containerName}" - - docker run \ - --rm \ - --detach \ - --name "${containerName}" \ - -v "${hostWorkspaceDir}":/workspace \ - -v "${HOME}/.cache/bazel":"${HOME}/.cache/bazel" \ - -v "${HOME}/.cache/shared_bazel_repository_cache":"${HOME}/.cache/shared_bazel_repository_cache" \ - -v "${HOME}/.cache/shared_bazel_action_cache":"${HOME}/.cache/shared_bazel_action_cache" \ - -v "${HOME}/.docker/config.json":"/home/builder/.docker/config.json" \ - -v "${HOME}/.aws":"/home/builder/.aws" \ - -v "/tmp/bazel-container/.bazelrc":"/etc/bazel.bazelrc" \ - --entrypoint=/bin/sleep \ - "${containerImage}" \ - infinity || return $? -} - -function stopBazelServer { - local containerName="bazeld" - - echo Stopping bazel container... - - docker kill "${containerName}" || return $? -} - -function bazel { - local containerName="bazeld" - - local hostWorkspaceDir - hostWorkspaceDir="$(git rev-parse --show-toplevel)" - if [[ $? -ne 0 ]]; then - echo Could not find git repository root. Are you in a git repository? - return 1 - fi - - local containerWorkDir - containerWorkDir=$(realpath -m "/workspace/$(realpath --relative-base="${hostWorkspaceDir}" .)") - if [[ $? -ne 0 ]]; then - echo Could not determine container work directory. - return 1 - fi - - docker exec \ - -it \ - --workdir "${containerWorkDir}" \ - --env "HOST_CACHE=${HOME}/.cache" \ - "${containerName}" \ - bazel "$@" || return $? -} diff --git a/bazel/devbuild/BUILD.bazel b/bazel/devbuild/BUILD.bazel index 27afb1347..12ac22edf 100644 --- a/bazel/devbuild/BUILD.bazel +++ b/bazel/devbuild/BUILD.bazel @@ -10,12 +10,18 @@ sh_template( "//bootstrapper/cmd/bootstrapper:bootstrapper_patched", "//cli:cli_edition_host", "//debugd/cmd/cdbg:cdbg_host", - "//terraform-provider-constellation:terraform_rc", - "//terraform-provider-constellation:tf_provider", "//upgrade-agent/cmd:upgrade_agent_linux_amd64", "@gnused//:bin/sed", "@yq_toolchains//:resolved_toolchain", - ], + ] + select( + { + "//bazel/settings:cli_edition_enterprise": [ + "//terraform-provider-constellation:terraform_rc", + "//terraform-provider-constellation:tf_provider", + ], + "//conditions:default": [], + }, + ), substitutions = { "@@BOOTSTRAPPER@@": "$(rootpath //bootstrapper/cmd/bootstrapper:bootstrapper_patched)", "@@CDBG@@": "$(rootpath //debugd/cmd/cdbg:cdbg_host)", @@ -23,8 +29,6 @@ sh_template( "@@CONTAINER_SUMS@@": "$(rootpath //bazel/release:container_sums)", "@@EDITION@@": "$(rootpath :devbuild_cli_edition)", "@@SED@@": "$(rootpath @gnused//:bin/sed)", - "@@TERRAFORM_PROVIDER@@": "$(rootpath //terraform-provider-constellation:tf_provider)", - "@@TERRAFORM_RC@@": "$(rootpath //terraform-provider-constellation:terraform_rc)", "@@UPGRADE_AGENT@@": "$(rootpath //upgrade-agent/cmd:upgrade_agent_linux_amd64)", "@@VERSION_FILE@@": "$(rootpath //bazel/settings:tag)", "@@YQ@@": "$(rootpath @yq_toolchains//:resolved_toolchain)", @@ -42,7 +46,18 @@ sh_template( "@platforms//cpu:x86_64": { "@@GOARCH@@": "amd64", }, - }), + }) | select( + { + "//bazel/settings:cli_edition_enterprise": { + "@@TERRAFORM_PROVIDER@@": "$(rootpath //terraform-provider-constellation:tf_provider)", + "@@TERRAFORM_RC@@": "$(rootpath //terraform-provider-constellation:terraform_rc)", + }, + "//conditions:default": { + "@@TERRAFORM_PROVIDER@@": "", + "@@TERRAFORM_RC@@": "", + }, + }, + ), template = "prepare_developer_workspace.sh.in", visibility = ["//visibility:public"], ) diff --git a/bazel/devbuild/prepare_developer_workspace.sh.in b/bazel/devbuild/prepare_developer_workspace.sh.in index 152a52230..0f9fe5d15 100755 --- a/bazel/devbuild/prepare_developer_workspace.sh.in +++ b/bazel/devbuild/prepare_developer_workspace.sh.in @@ -38,8 +38,13 @@ stat "${cdbg}" >> /dev/null container_sums=$(realpath @@CONTAINER_SUMS@@) stat "${container_sums}" >> /dev/null edition=$(cat @@EDITION@@) -terraform_provider=$(realpath @@TERRAFORM_PROVIDER@@) -stat "${terraform_provider}" >> /dev/null +raw_provider="@@TERRAFORM_PROVIDER@@" +if [[ -n ${raw_provider} ]]; then + terraform_provider=$(realpath "${raw_provider}") + stat "${terraform_provider}" >> /dev/null +else + terraform_provider="" +fi build_version=$(cat @@VERSION_FILE@@) if [[ -z ${build_version} ]]; then echo "Error: version file is empty" @@ -80,9 +85,11 @@ ln -sf "$(replace_prefix "${host_cache}" "${builder_cache}" "${cdbg}")" "${workd ln -sf "$(replace_prefix "${host_cache}" "${builder_cache}" "${container_sums}")" "${workdir}/container_sums.sha256" ln -sf "$(replace_prefix "${host_cache}" "${builder_cache}" "${cli}")" "${workdir}/constellation" -terraform_provider_dir=${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/${goos}_${goarch}/ -mkdir -p "${terraform_provider_dir}" -ln -sf "${terraform_provider}" "${terraform_provider_dir}/terraform-provider-constellation_${build_version}" +if [[ -n ${terraform_provider} ]]; then + terraform_provider_dir=${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/${goos}_${goarch}/ + mkdir -p "${terraform_provider_dir}" + ln -sf "${terraform_provider}" "${terraform_provider_dir}/terraform-provider-constellation_${build_version}" +fi if [[ ! -f "${workdir}/constellation-conf.yaml" ]]; then echo "constellation-conf.yaml not present in workspace" diff --git a/bazel/go/go.nix b/bazel/go/go.nix deleted file mode 100644 index 9a3985be9..000000000 --- a/bazel/go/go.nix +++ /dev/null @@ -1,21 +0,0 @@ -let - pkgs = import { }; - goAttr = pkgs.go_1_21.overrideAttrs (_: rec { - version = "1.21.5"; - src = pkgs.fetchurl { - url = "https://go.dev/dl/go${version}.src.tar.gz"; - hash = "sha256-KFy730tubmLtWPNw8/bYwwgl1uVsWFPGbTwjvNsJ2xk="; - }; - }); -in -pkgs.buildEnv - { - name = "bazel-go-toolchain"; - paths = [ goAttr ]; - postBuild = '' - touch $out/ROOT - ln -s $out/share/go/{api,doc,lib,misc,pkg,src,go.env} $out/ - ''; - } // { - version = goAttr.version; -} diff --git a/bazel/mkosi/mkosi_image.bzl b/bazel/mkosi/mkosi_image.bzl index c917c09ad..fb11a81c2 100644 --- a/bazel/mkosi/mkosi_image.bzl +++ b/bazel/mkosi/mkosi_image.bzl @@ -1,40 +1,43 @@ """ Bazel rule for building mkosi images. """ -load("@aspect_bazel_lib//lib:paths.bzl", "relative_file") load("@bazel_skylib//lib:paths.bzl", "paths") +def _resource_set(_os, _num_inputs): + return {"cpu": 4, "memory": 4096} + def _mkosi_image_impl(ctx): args = ctx.actions.args() inputs = [] outputs = [] tools = [] workdir = ctx.file.mkosi_conf.dirname - config_rel = lambda target: relative_file(target, ctx.file.mkosi_conf.path) env = {} args.add("-C", workdir) if ctx.attr.distribution: args.add("--distribution", ctx.attr.distribution) if ctx.attr.architecture: args.add("--architecture", ctx.attr.architecture) + if ctx.attr.output: + args.add("--output", ctx.attr.output) args.add_all(ctx.attr.packages, before_each = "--package") - for package_file in ctx.files.package_files: - args.add("--package", config_rel(package_file.path)) + for package_dir in ctx.files.package_directories: + args.add("--package-directory", package_dir.path) if len(ctx.files.local_mirror) > 0: - env["LOCAL_MIRROR"] = config_rel(ctx.files.local_mirror[0].dirname) + env["LOCAL_MIRROR"] = ctx.files.local_mirror[0].dirname for tree in ctx.files.base_trees: - args.add("--base-tree", config_rel(tree.path)) + args.add("--base-tree", tree.path) for tree in ctx.files.skeleton_trees: - args.add("--skeleton-tree", config_rel(tree.path)) + args.add("--skeleton-tree", tree.path) for tree in ctx.files.package_manager_trees: - args.add("--package-manager_tree", config_rel(tree.path)) + args.add("--package-manager-tree", tree.path) for tree in ctx.files.extra_trees: - args.add("--extra-tree", config_rel(tree.path)) + args.add("--extra-tree", tree.path) for initrd in ctx.files.initrds: inputs.append(initrd) - args.add("--initrd", config_rel(initrd.path)) + args.add("--initrd", initrd.path) inputs.extend(ctx.files.mkosi_conf) inputs.extend(ctx.files.srcs[:]) - inputs.extend(ctx.files.package_files[:]) + inputs.extend(ctx.files.package_directories[:]) inputs.extend(ctx.files.base_trees[:]) inputs.extend(ctx.files.skeleton_trees[:]) inputs.extend(ctx.files.package_manager_trees[:]) @@ -57,9 +60,9 @@ def _mkosi_image_impl(ctx): if output.is_directory and out_dir.path.startswith(output.path + "/"): fail("output directory {} is nested within output directory {}; outputs cannot be nested within each other!".format(out_dir.path, output.path)) outputs.append(out_dir) - args.add("--output-dir", config_rel(out_dir.path)) + args.add("--output-dir", out_dir.path) else: - args.add("--output-dir", config_rel(paths.join(ctx.bin_dir.path, ctx.label.package))) + args.add("--output-dir", paths.join(ctx.bin_dir.path, ctx.label.package)) args.add_all(ctx.attr.extra_args) for key, value in ctx.attr.env.items(): args.add("--environment", "{}={}".format(key, value)) @@ -67,8 +70,6 @@ def _mkosi_image_impl(ctx): args.add("--kernel-command-line", ctx.attr.kernel_command_line) for key, value in ctx.attr.kernel_command_line_dict.items(): args.add("--kernel-command-line", "{}={}".format(key, value)) - if ctx.attr.autologin: - args.add("--autologin", "yes") info = ctx.toolchains["@constellation//bazel/mkosi:toolchain_type"].mkosi if not info.valid: @@ -99,6 +100,7 @@ def _mkosi_image_impl(ctx): execution_requirements = {"no-remote": "1", "no-sandbox": "1"}, progress_message = "Building mkosi image " + ctx.label.name, env = env, + resource_set = _resource_set, ) return DefaultInfo(files = depset(outputs), runfiles = ctx.runfiles(outputs)) @@ -106,7 +108,6 @@ mkosi_image = rule( implementation = _mkosi_image_impl, attrs = { "architecture": attr.string(), - "autologin": attr.bool(), "base_trees": attr.label_list(allow_files = True), "distribution": attr.string(), "env": attr.string_dict(), @@ -122,8 +123,9 @@ mkosi_image = rule( doc = "main mkosi.conf file", ), "out_dir": attr.string(), + "output": attr.string(), "outs": attr.output_list(), - "package_files": attr.label_list(allow_files = True), + "package_directories": attr.label_list(allow_files = True), "package_manager_trees": attr.label_list(allow_files = True), "packages": attr.string_list(), "seed": attr.string(), diff --git a/bazel/mkosi/mkosi_wrapper.sh.in b/bazel/mkosi/mkosi_wrapper.sh.in index c8e541a0b..370533a38 100644 --- a/bazel/mkosi/mkosi_wrapper.sh.in +++ b/bazel/mkosi/mkosi_wrapper.sh.in @@ -4,7 +4,24 @@ shopt -s inherit_errexit export PATH=/run/wrappers/bin:/run/current-system/sw/bin:/bin:/usr/bin:/usr/local/bin VERSION_ARG="" -args=("$@") +args=() + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --*-tree | --initrd | --package-directory | --output-dir) + # absolutize any file paths + shift # past the key and to the value + value="$1" + args+=("${key}" "$(realpath "${value}")") + shift # past the value + ;; + *) + args+=("$1") + shift + ;; + esac +done if [[ -n ${VERSION_FILE+x} ]]; then VERSION_ARG="--environment=IMAGE_VERSION=$(cat "${VERSION_FILE}")" @@ -14,14 +31,12 @@ fi if [[ -n ${LOCAL_MIRROR+x} ]]; then LOCAL_MIRROR=$(realpath "${LOCAL_MIRROR}") reposdir=$(mktemp -d) - cat > "${reposdir}/mkosi.repo" << EOF -[local-mirror] -name=local-mirror -baseurl=file://${LOCAL_MIRROR} -enabled=1 -gpgcheck=0 -EOF + # putting an empty repo file under /etc/yum.repos.d/mkosi.repo + # will make mkosi use only package directories + # and not try to fetch packages from the network + touch "${reposdir}/mkosi.repo" args+=("--package-manager-tree=${reposdir}:/etc/yum.repos.d") + args+=("--package-directory" "${LOCAL_MIRROR}") fi exec @@MKOSI@@ "${args[@]}" build diff --git a/bazel/oci/containers.bzl b/bazel/oci/containers.bzl index b83ef8262..3e56ba99a 100644 --- a/bazel/oci/containers.bzl +++ b/bazel/oci/containers.bzl @@ -63,6 +63,14 @@ def containers(): "repotag_file": "//bazel/release:s3proxy_tag.txt", "used_by": ["config"], }, + { + "identifier": "vpn", + "image_name": "vpn", + "name": "vpn", + "oci": "//nix/container/vpn", + "repotag_file": "//bazel/release:vpn_tag.txt", + "used_by": ["config"], + }, ] def helm_containers(): diff --git a/bazel/osimage/BUILD.bazel b/bazel/osimage/BUILD.bazel new file mode 100644 index 000000000..1953012ca --- /dev/null +++ b/bazel/osimage/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["upload_os_images.sh.in"]) diff --git a/bazel/osimage/upload_os_images.bzl b/bazel/osimage/upload_os_images.bzl new file mode 100644 index 000000000..f3784103e --- /dev/null +++ b/bazel/osimage/upload_os_images.bzl @@ -0,0 +1,104 @@ +""" Bazel rule for uploading a set of OS images to cloud providers. """ + +def _upload_os_images_impl(ctx): + executable = ctx.actions.declare_file("upload_os_images_%s.sh" % ctx.label.name) + files = [] + files.extend(ctx.files.image_dirs) + files.append(ctx.file._version) + files.append(ctx.file._upload_cli) + files.append(ctx.file._measured_boot) + files.append(ctx.file._uplosi) + files.append(ctx.file._dissect_toolchain) + files.append(ctx.file._cosign) + files.append(ctx.file._rekor_cli) + files.append(ctx.file._parallel) + raw_image_paths = [] + for image_dir in ctx.files.image_dirs: + raw_image_paths.append("%s/constellation.raw" % image_dir.short_path) + substitutions = { + "@@COSIGN@@": ctx.executable._cosign.short_path, + "@@DISSECT_TOOLCHAIN@@": ctx.executable._dissect_toolchain.short_path, + "@@FILES@@": " ".join(raw_image_paths), + "@@MEASURED_BOOT@@": ctx.executable._measured_boot.short_path, + "@@PARALLEL@@": ctx.executable._parallel.short_path, + "@@REKOR_CLI@@": ctx.executable._rekor_cli.short_path, + "@@UPLOAD_CLI@@": ctx.executable._upload_cli.short_path, + "@@UPLOSI@@": ctx.executable._uplosi.short_path, + "@@VERSION@@": ctx.file._version.short_path, + } + ctx.actions.expand_template( + template = ctx.file._upload_sh_tpl, + output = executable, + is_executable = True, + substitutions = substitutions, + ) + runfiles = ctx.runfiles(files = files) + runfiles = runfiles.merge(ctx.attr._uplosi[DefaultInfo].data_runfiles) + runfiles = runfiles.merge(ctx.attr._dissect_toolchain[DefaultInfo].data_runfiles) + runfiles = runfiles.merge(ctx.attr._cosign[DefaultInfo].data_runfiles) + runfiles = runfiles.merge(ctx.attr._rekor_cli[DefaultInfo].data_runfiles) + runfiles = runfiles.merge(ctx.attr._parallel[DefaultInfo].data_runfiles) + runfiles = runfiles.merge(ctx.attr._upload_cli[DefaultInfo].data_runfiles) + runfiles = runfiles.merge(ctx.attr._measured_boot[DefaultInfo].data_runfiles) + + return DefaultInfo(executable = executable, runfiles = runfiles) + +upload_os_images = rule( + implementation = _upload_os_images_impl, + attrs = { + "image_dirs": attr.label_list( + doc = "List of directories containing OS images to upload.", + ), + "_cosign": attr.label( + default = Label("@cosign//:bin/cosign"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_dissect_toolchain": attr.label( + default = Label("@systemd//:bin/systemd-dissect"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_measured_boot": attr.label( + default = Label("//image/measured-boot/cmd"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_parallel": attr.label( + default = Label("@parallel//:bin/parallel"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_rekor_cli": attr.label( + default = Label("@rekor-cli//:bin/rekor-cli"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_upload_cli": attr.label( + default = Label("//image/upload"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_upload_sh_tpl": attr.label( + default = "upload_os_images.sh.in", + allow_single_file = True, + ), + "_uplosi": attr.label( + default = Label("@uplosi//:bin/uplosi"), + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "_version": attr.label( + default = Label("//bazel/settings:tag"), + allow_single_file = True, + ), + }, + executable = True, +) diff --git a/bazel/osimage/upload_os_images.sh.in b/bazel/osimage/upload_os_images.sh.in new file mode 100644 index 000000000..08f1c04ab --- /dev/null +++ b/bazel/osimage/upload_os_images.sh.in @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +set -euo pipefail +shopt -s inherit_errexit + +# This script handles the upload of OS images and their corresponding image info. + +POSITIONAL_ARGS=() + +ref="" +upload_signed_measurements=0 +fake_sign=0 + +while [[ $# -gt 0 ]]; do + case $1 in + --ref) + ref="$2" + shift # past argument + shift # past value + ;; + --upload-measurements) + upload_signed_measurements=1 + shift # past argument + ;; + --fake-sign) + fake_sign=1 + shift # past argument + ;; + -*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + +if [[ $# -ne 0 ]]; then + echo "Unknown positional arguments: $*" + exit 1 +fi + +if [[ -z ${ref} ]]; then + echo "Missing required argument --ref" + exit 1 +fi + +version_file=$(realpath @@VERSION@@) +stat "${version_file}" >> /dev/null +version=$(cat "${version_file}") + +uplosi=$(realpath @@UPLOSI@@) +stat "${uplosi}" >> /dev/null + +systemd_dissect=$(realpath @@DISSECT_TOOLCHAIN@@) +stat "${systemd_dissect}" >> /dev/null +export DISSECT_TOOLCHAIN="${systemd_dissect}" + +cosign=$(realpath @@COSIGN@@) +stat "${cosign}" >> /dev/null + +rekor_cli=$(realpath @@REKOR_CLI@@) +stat "${rekor_cli}" >> /dev/null + +upload_cli=$(realpath @@UPLOAD_CLI@@) +stat "${upload_cli}" >> /dev/null + +measured_boot=$(realpath @@MEASURED_BOOT@@) +stat "${measured_boot}" >> /dev/null + +parallel=$(realpath @@PARALLEL@@) +stat "${parallel}" >> /dev/null + +FILES=(@@FILES@@) + +workspace=$(mktemp -d) +# shellcheck disable=SC2064 +trap "rm -rf ${workspace}" EXIT + +echo Uploading "${#FILES[@]}" OS images. This may take a while... >&2 + +"${parallel}" --will-cite \ + "${upload_cli}" uplosi \ + --uplosi-path "${uplosi}" \ + --version "${version}" \ + --ref "${ref}" \ + --raw-image {} \ + --out "${workspace}/image-upload-{#}.json" \ + ::: "${FILES[@]}" + +"${upload_cli}" info "${workspace}/"image-upload-*.json + +if [[ ${upload_signed_measurements} -eq 0 ]]; then + echo "Skipping signed measurements upload. Enable by setting --upload-measurements" >&2 + exit 0 +fi + +echo Uploading signed measurements. This requires sudo and a signing key. >&2 +i=1 +for file in "${FILES[@]}"; do + combined_name=$(basename "$(dirname "${file}")") + IFS="_" read -r csp attestation_variant stream <<< "${combined_name}" + sudo -E "${measured_boot}" "${file}" "${workspace}/pcrs-${i}.json" + sudo chown "$(id -u -n)" "${workspace}/pcrs-${i}.json" + "${upload_cli}" measurements envelope \ + --in "${workspace}/pcrs-${i}.json" \ + --out "${workspace}/pcrs-${i}.json" \ + --version "ref/${ref}/stream/${stream}/${version}" \ + --csp "${csp}" \ + --attestation-variant "${attestation_variant}" + i=$((i + 1)) +done + +"${upload_cli}" measurements merge \ + --out "${workspace}/measurements.json" \ + "${workspace}"/pcrs-*.json + +if [[ ${fake_sign} -eq 1 ]]; then + echo "Skipping signing of measurements and using fake signature instead (--fake-sign is set)." >&2 + echo "THOSE MEASUREMENTS BELONG TO A DEBUG IMAGE. THOSE ARE NOT SINGED BY ANY KEY." > "${workspace}/measurements.json.sig" +else + # shellcheck disable=SC2016 + echo 'Creating real signature with keys referenced in $COSIGN_PUBLIC_KEY_PATH, $COSIGN_PRIVATE_KEY and $COSIGN_PASSWORD. Set "--fake-sign" for debugging purposes.' >&2 + # Enabling experimental mode also publishes signature to Rekor + COSIGN_EXPERIMENTAL=1 "${cosign}" sign-blob --yes --key env://COSIGN_PRIVATE_KEY \ + "${workspace}/measurements.json" > "${workspace}/measurements.json.sig" + # Verify - As documentation & check + # Local Signature (input: artifact, key, signature) + "${cosign}" verify-blob --key "${COSIGN_PUBLIC_KEY_PATH}" \ + --signature "${workspace}/measurements.json.sig" \ + "${workspace}/measurements.json" + # Transparency Log Signature (input: artifact, key) + uuid=$("${rekor_cli}" search --artifact "${workspace}/measurements.json" | tail -n 1) + sig=$("${rekor_cli}" get --uuid="${uuid}" --format=json | jq -r .Body.HashedRekordObj.signature.content) + "${cosign}" verify-blob --key "${COSIGN_PUBLIC_KEY_PATH}" --signature <(echo "${sig}") "${workspace}/measurements.json" +fi + +"${upload_cli}" measurements upload \ + --measurements "${workspace}/measurements.json" \ + --signature "${workspace}/measurements.json.sig" diff --git a/bazel/proto/rules.bzl b/bazel/proto/rules.bzl index da73d5f72..3807796b2 100644 --- a/bazel/proto/rules.bzl +++ b/bazel/proto/rules.bzl @@ -5,17 +5,14 @@ based on https://github.com/bazelbuild/rules_go/issues/2111#issuecomment-1355927 """ load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files") -load("@io_bazel_rules_go//go:def.bzl", "GoLibrary", "go_context") +load("@io_bazel_rules_go//go:def.bzl", "GoInfo") load("@io_bazel_rules_go//proto:compiler.bzl", "GoProtoCompiler") def _output_go_library_srcs_impl(ctx): - go = go_context(ctx) - srcs_of_library = [] importpath = "" for src in ctx.attr.deps: - lib = src[GoLibrary] - go_src = go.library_to_source(go, ctx.attr, lib, False) + lib = src[GoInfo] if importpath and lib.importpath != importpath: fail( "importpath of all deps must match, got {} and {}", @@ -23,7 +20,7 @@ def _output_go_library_srcs_impl(ctx): lib.importpath, ) importpath = lib.importpath - srcs_of_library.extend(go_src.srcs) + srcs_of_library.extend(lib.srcs) if len(srcs_of_library) != 1: fail("expected exactly one src for library, got {}", len(srcs_of_library)) @@ -54,7 +51,7 @@ output_go_library_srcs = rule( default = "@io_bazel_rules_go//proto:go_proto", ), "deps": attr.label_list( - providers = [GoLibrary], + providers = [GoInfo], aspects = [], ), "out": attr.output( diff --git a/bazel/release/BUILD.bazel b/bazel/release/BUILD.bazel index 23738421d..fca8a017d 100644 --- a/bazel/release/BUILD.bazel +++ b/bazel/release/BUILD.bazel @@ -29,16 +29,6 @@ load("//bazel/oci:pin.bzl", "oci_sum", "oci_sum_merge") for container in containers() ] -# TODO(3u13r): re-enable target once https://github.com/bazel-contrib/rules_oci/issues/184 is fixed -# [ -# oci_tarball( -# name = container["name"] + "_tar", -# image = container["oci"], -# repotag_file = container["repotag_file"], -# ) -# for container in containers() -# ] - [ oci_sum( name = container["name"] + "_sum", @@ -62,7 +52,8 @@ oci_sum_merge( oci_push( name = container["name"] + "_push", image = container["oci"], - repotags = container["repotag_file"], + remote_tags = "//bazel/settings:tag", + repository_file = "//bazel/release:" + container["name"] + "_reponame", ) for container in containers() ] diff --git a/bazel/release/artifacts/BUILD.bazel b/bazel/release/artifacts/BUILD.bazel index bba7fb0c8..8861d6dfe 100644 --- a/bazel/release/artifacts/BUILD.bazel +++ b/bazel/release/artifacts/BUILD.bazel @@ -70,5 +70,5 @@ go_test( env = platform_container_sums_paths | platform_clis_paths, # keep x_defs = {"runsUnder": "bazel"}, - deps = ["@io_bazel_rules_go//go/runfiles:go_default_library"], + deps = ["@io_bazel_rules_go//go/runfiles"], ) diff --git a/bazel/release/artifacts/artifacts_test.go b/bazel/release/artifacts/artifacts_test.go index 3a093d21a..0c23a3f41 100644 --- a/bazel/release/artifacts/artifacts_test.go +++ b/bazel/release/artifacts/artifacts_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package artifacts diff --git a/bazel/sh/BUILD.bazel b/bazel/sh/BUILD.bazel index cac8fda5a..7a347a852 100644 --- a/bazel/sh/BUILD.bazel +++ b/bazel/sh/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_shell//shell:sh_library.bzl", "sh_library") + exports_files([ "repo_command.sh.in", "noop_warn.sh.in", diff --git a/bazel/sh/def.bzl b/bazel/sh/def.bzl index 5f2129017..4ddad1fe0 100644 --- a/bazel/sh/def.bzl +++ b/bazel/sh/def.bzl @@ -1,6 +1,8 @@ """Bazel rules for CI and dev tooling""" load("@bazel_skylib//lib:shell.bzl", "shell") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("@rules_shell//shell:sh_test.bzl", "sh_test") def _sh_template_impl(ctx): out_file = ctx.actions.declare_file(ctx.label.name + ".bash") @@ -66,8 +68,7 @@ def sh_template(name, **kwargs): template = template, toolchains = toolchains, ) - - native.sh_binary( + sh_binary( name = name, srcs = [script_name], data = [script_name] + data, @@ -95,8 +96,7 @@ def sh_test_template(name, **kwargs): substitutions = substitutions, template = template, ) - - native.sh_test( + sh_test( name = name, srcs = [script_name], data = [script_name] + data, diff --git a/bazel/toolchains/0001-disable-Windows-support.patch b/bazel/toolchains/0001-disable-Windows-support.patch new file mode 100644 index 000000000..1cb9c8340 --- /dev/null +++ b/bazel/toolchains/0001-disable-Windows-support.patch @@ -0,0 +1,75 @@ +From d10473f4ac89c23dcd8ea02488b28a649f4a9735 Mon Sep 17 00:00:00 2001 +From: Markus Rudy +Date: Tue, 6 Aug 2024 11:33:29 +0200 +Subject: [PATCH] disable Windows support + +It's broken and we don't need it, see +https://github.com/bazel-contrib/rules_oci/issues/420. +--- + oci/private/image.bzl | 9 --------- + oci/private/util.bzl | 29 +---------------------------- + 2 files changed, 1 insertion(+), 37 deletions(-) + +diff --git a/oci/private/image.bzl b/oci/private/image.bzl +index e8a6ca5..434947c 100644 +--- a/oci/private/image.bzl ++++ b/oci/private/image.bzl +@@ -226,15 +226,6 @@ def _oci_image_impl(ctx): + + action_env = {} + +- # Windows: Don't convert arguments like --entrypoint=/some/bin to --entrypoint=C:/msys64/some/bin +- if ctx.target_platform_has_constraint(ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]): +- # See https://www.msys2.org/wiki/Porting/: +- # > Setting MSYS2_ARG_CONV_EXCL=* prevents any path transformation. +- action_env["MSYS2_ARG_CONV_EXCL"] = "*" +- +- # This one is for Windows Git MSys +- action_env["MSYS_NO_PATHCONV"] = "1" +- + ctx.actions.run( + inputs = depset(inputs, transitive = transitive_inputs), + arguments = [args], +diff --git a/oci/private/util.bzl b/oci/private/util.bzl +index 7c2a2c2..479ca7d 100644 +--- a/oci/private/util.bzl ++++ b/oci/private/util.bzl +@@ -141,34 +141,7 @@ def _maybe_wrap_launcher_for_windows(ctx, bash_launcher): + - make sure the bash_launcher is in the inputs to the action + - @bazel_tools//tools/sh:toolchain_type should appear in the rules toolchains + """ +- if not ctx.target_platform_has_constraint(ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]): +- return bash_launcher +- +- win_launcher = ctx.actions.declare_file("wrap_%s.bat" % ctx.label.name) +- ctx.actions.write( +- output = win_launcher, +- content = r"""@echo off +-SETLOCAL ENABLEEXTENSIONS +-SETLOCAL ENABLEDELAYEDEXPANSION +-for %%a in ("{bash_bin}") do set "bash_bin_dir=%%~dpa" +-set PATH=%bash_bin_dir%;%PATH% +-set "parent_dir=%~dp0" +-set "parent_dir=!parent_dir:\=/!" +-set args=%* +-rem Escape \ and * in args before passing it with double quote +-if defined args ( +- set args=!args:\=\\\\! +- set args=!args:"=\"! +-) +-"{bash_bin}" -c "%parent_dir%{launcher} !args!" +-""".format( +- bash_bin = ctx.toolchains["@bazel_tools//tools/sh:toolchain_type"].path, +- launcher = paths.relativize(bash_launcher.path, win_launcher.dirname), +- ), +- is_executable = True, +- ) +- +- return win_launcher ++ return bash_launcher + + def _file_exists(rctx, path): + result = rctx.execute(["stat", path]) +-- +2.46.0 + diff --git a/bazel/toolchains/BUILD.bazel b/bazel/toolchains/BUILD.bazel index 5849b1b8b..55a19db8a 100644 --- a/bazel/toolchains/BUILD.bazel +++ b/bazel/toolchains/BUILD.bazel @@ -1 +1 @@ -"""This folder contains toolchain dependencies for the project. They are loaded by `WORKSPACE.bazel`.""" +"""This folder contains toolchain dependencies for the project. They are loaded by `WORKSPACE.bzlmod`.""" diff --git a/bazel/toolchains/aspect_bazel_lib.bzl b/bazel/toolchains/aspect_bazel_lib.bzl deleted file mode 100644 index b9b7feb9a..000000000 --- a/bazel/toolchains/aspect_bazel_lib.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""aspect bazel library""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def aspect_bazel_lib(): - http_archive( - name = "aspect_bazel_lib", - sha256 = "4b32cf6feab38b887941db022020eea5a49b848e11e3d6d4d18433594951717a", - strip_prefix = "bazel-lib-2.0.1", - urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/4b32cf6feab38b887941db022020eea5a49b848e11e3d6d4d18433594951717a", - "https://github.com/aspect-build/bazel-lib/releases/download/v2.0.1/bazel-lib-v2.0.1.tar.gz", - ], - type = "tar.gz", - ) diff --git a/bazel/toolchains/buildifier_deps.bzl b/bazel/toolchains/buildifier_deps.bzl deleted file mode 100644 index 222895524..000000000 --- a/bazel/toolchains/buildifier_deps.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""buildifier repository rules""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def buildifier_deps(): - http_archive( - name = "com_github_bazelbuild_buildtools", - strip_prefix = "buildtools-6.3.3", - urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/42968f9134ba2c75c03bb271bd7bb062afb7da449f9b913c96e5be4ce890030a", - "https://github.com/bazelbuild/buildtools/archive/refs/tags/v6.3.3.tar.gz", - ], - type = "tar.gz", - sha256 = "42968f9134ba2c75c03bb271bd7bb062afb7da449f9b913c96e5be4ce890030a", - ) diff --git a/bazel/toolchains/ci_deps.bzl b/bazel/toolchains/ci_deps.bzl index 4cf214096..f6354e6ca 100644 --- a/bazel/toolchains/ci_deps.bzl +++ b/bazel/toolchains/ci_deps.bzl @@ -19,33 +19,33 @@ def _shellcheck_deps(): http_archive( name = "com_github_koalaman_shellcheck_linux_amd64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/700324c6dd0ebea0117591c6cc9d7350d9c7c5c287acbad7630fa17b1d4d9e2f", - "https://github.com/koalaman/shellcheck/releases/download/v0.9.0/shellcheck-v0.9.0.linux.x86_64.tar.xz", + "https://cdn.confidential.cloud/constellation/cas/sha256/6c881ab0698e4e6ea235245f22832860544f17ba386442fe7e9d629f8cbedf87", + "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz", ], - sha256 = "700324c6dd0ebea0117591c6cc9d7350d9c7c5c287acbad7630fa17b1d4d9e2f", - strip_prefix = "shellcheck-v0.9.0", + sha256 = "6c881ab0698e4e6ea235245f22832860544f17ba386442fe7e9d629f8cbedf87", + strip_prefix = "shellcheck-v0.10.0", build_file_content = """exports_files(["shellcheck"], visibility = ["//visibility:public"])""", type = "tar.xz", ) http_archive( name = "com_github_koalaman_shellcheck_linux_arm64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/179c579ef3481317d130adebede74a34dbbc2df961a70916dd4039ebf0735fae", - "https://github.com/koalaman/shellcheck/releases/download/v0.9.0/shellcheck-v0.9.0.linux.aarch64.tar.xz", + "https://cdn.confidential.cloud/constellation/cas/sha256/324a7e89de8fa2aed0d0c28f3dab59cf84c6d74264022c00c22af665ed1a09bb", + "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.aarch64.tar.xz", ], - sha256 = "179c579ef3481317d130adebede74a34dbbc2df961a70916dd4039ebf0735fae", - strip_prefix = "shellcheck-v0.9.0", + sha256 = "324a7e89de8fa2aed0d0c28f3dab59cf84c6d74264022c00c22af665ed1a09bb", + strip_prefix = "shellcheck-v0.10.0", build_file_content = """exports_files(["shellcheck"], visibility = ["//visibility:public"])""", type = "tar.xz", ) http_archive( name = "com_github_koalaman_shellcheck_darwin_amd64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/7d3730694707605d6e60cec4efcb79a0632d61babc035aa16cda1b897536acf5", - "https://github.com/koalaman/shellcheck/releases/download/v0.9.0/shellcheck-v0.9.0.darwin.x86_64.tar.xz", + "https://cdn.confidential.cloud/constellation/cas/sha256/ef27684f23279d112d8ad84e0823642e43f838993bbb8c0963db9b58a90464c2", + "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.darwin.x86_64.tar.xz", ], - sha256 = "7d3730694707605d6e60cec4efcb79a0632d61babc035aa16cda1b897536acf5", - strip_prefix = "shellcheck-v0.9.0", + sha256 = "ef27684f23279d112d8ad84e0823642e43f838993bbb8c0963db9b58a90464c2", + strip_prefix = "shellcheck-v0.10.0", build_file_content = """exports_files(["shellcheck"], visibility = ["//visibility:public"])""", type = "tar.xz", ) @@ -55,40 +55,40 @@ def _terraform_deps(): name = "com_github_hashicorp_terraform_linux_amd64", build_file_content = """exports_files(["terraform"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/9f3ca33d04f5335472829d1df7785115b60176d610ae6f1583343b0a2221a931", - "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_amd64.zip", + "https://cdn.confidential.cloud/constellation/cas/sha256/ad0c696c870c8525357b5127680cd79c0bdf58179af9acd091d43b1d6482da4a", + "https://releases.hashicorp.com/terraform/1.5.5/terraform_1.5.5_linux_amd64.zip", ], - sha256 = "9f3ca33d04f5335472829d1df7785115b60176d610ae6f1583343b0a2221a931", + sha256 = "ad0c696c870c8525357b5127680cd79c0bdf58179af9acd091d43b1d6482da4a", type = "zip", ) http_archive( name = "com_github_hashicorp_terraform_linux_arm64", build_file_content = """exports_files(["terraform"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/39c182670c4e63e918e0a16080b1cc47bb16e158d7da96333d682d6a9cb8eb91", - "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_arm64.zip", + "https://cdn.confidential.cloud/constellation/cas/sha256/b055aefe343d0b710d8a7afd31aeb702b37bbf4493bb9385a709991e48dfbcd2", + "https://releases.hashicorp.com/terraform/1.5.5/terraform_1.5.5_linux_arm64.zip", ], - sha256 = "39c182670c4e63e918e0a16080b1cc47bb16e158d7da96333d682d6a9cb8eb91", + sha256 = "b055aefe343d0b710d8a7afd31aeb702b37bbf4493bb9385a709991e48dfbcd2", type = "zip", ) http_archive( name = "com_github_hashicorp_terraform_darwin_amd64", build_file_content = """exports_files(["terraform"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/c218a6c0ef6692b25af16995c8c7bdf6739e9638fef9235c6aced3cd84afaf66", - "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_darwin_amd64.zip", + "https://cdn.confidential.cloud/constellation/cas/sha256/6d61639e2141b7c23a9219c63994f729aa41f91110a1aa08b8a37969fb45e229", + "https://releases.hashicorp.com/terraform/1.5.5/terraform_1.5.5_darwin_amd64.zip", ], - sha256 = "c218a6c0ef6692b25af16995c8c7bdf6739e9638fef9235c6aced3cd84afaf66", + sha256 = "6d61639e2141b7c23a9219c63994f729aa41f91110a1aa08b8a37969fb45e229", type = "zip", ) http_archive( name = "com_github_hashicorp_terraform_darwin_arm64", build_file_content = """exports_files(["terraform"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/af8ff7576c8fc41496fdf97e9199b00d8d81729a6a0e821eaf4dfd08aa763540", - "https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_darwin_arm64.zip", + "https://cdn.confidential.cloud/constellation/cas/sha256/c7fdeddb4739fdd5bada9d45fd786e2cbaf6e9e364693eee45c83e95281dad3a", + "https://releases.hashicorp.com/terraform/1.5.5/terraform_1.5.5_darwin_arm64.zip", ], - sha256 = "af8ff7576c8fc41496fdf97e9199b00d8d81729a6a0e821eaf4dfd08aa763540", + sha256 = "c7fdeddb4739fdd5bada9d45fd786e2cbaf6e9e364693eee45c83e95281dad3a", type = "zip", ) @@ -97,83 +97,83 @@ def _actionlint_deps(): name = "com_github_rhysd_actionlint_linux_amd64", build_file_content = """exports_files(["actionlint"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/f0294c342af98fad4ff917bc32032f28e1b55f76aedf291886ec10bbed7c12e1", - "https://github.com/rhysd/actionlint/releases/download/v1.6.26/actionlint_1.6.26_linux_amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757", + "https://github.com/rhysd/actionlint/releases/download/v1.7.7/actionlint_1.7.7_linux_amd64.tar.gz", ], type = "tar.gz", - sha256 = "f0294c342af98fad4ff917bc32032f28e1b55f76aedf291886ec10bbed7c12e1", + sha256 = "023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757", ) http_archive( name = "com_github_rhysd_actionlint_linux_arm64", build_file_content = """exports_files(["actionlint"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/a1056d85d614af4f6e5517ed2911dab2621b8e97c368c8b265328f9c22801648", - "https://github.com/rhysd/actionlint/releases/download/v1.6.26/actionlint_1.6.26_linux_arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/401942f9c24ed71e4fe71b76c7d638f66d8633575c4016efd2977ce7c28317d0", + "https://github.com/rhysd/actionlint/releases/download/v1.7.7/actionlint_1.7.7_linux_arm64.tar.gz", ], type = "tar.gz", - sha256 = "a1056d85d614af4f6e5517ed2911dab2621b8e97c368c8b265328f9c22801648", + sha256 = "401942f9c24ed71e4fe71b76c7d638f66d8633575c4016efd2977ce7c28317d0", ) http_archive( name = "com_github_rhysd_actionlint_darwin_amd64", build_file_content = """exports_files(["actionlint"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/bfa890e77a8508603c785af09a30bbab3a3255d291d8d27efc3f20ac8e303a8e", - "https://github.com/rhysd/actionlint/releases/download/v1.6.26/actionlint_1.6.26_darwin_amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/28e5de5a05fc558474f638323d736d822fff183d2d492f0aecb2b73cc44584f5", + "https://github.com/rhysd/actionlint/releases/download/v1.7.7/actionlint_1.7.7_darwin_amd64.tar.gz", ], type = "tar.gz", - sha256 = "bfa890e77a8508603c785af09a30bbab3a3255d291d8d27efc3f20ac8e303a8e", + sha256 = "28e5de5a05fc558474f638323d736d822fff183d2d492f0aecb2b73cc44584f5", ) http_archive( name = "com_github_rhysd_actionlint_darwin_arm64", build_file_content = """exports_files(["actionlint"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/5e131ab7de7ad051e1923b80d167aaa414734e97c720698c48778250e1dd2590", - "https://github.com/rhysd/actionlint/releases/download/v1.6.26/actionlint_1.6.26_darwin_arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/2693315b9093aeacb4ebd91a993fea54fc215057bf0da2659056b4bc033873db", + "https://github.com/rhysd/actionlint/releases/download/v1.7.7/actionlint_1.7.7_darwin_arm64.tar.gz", ], type = "tar.gz", - sha256 = "5e131ab7de7ad051e1923b80d167aaa414734e97c720698c48778250e1dd2590", + sha256 = "2693315b9093aeacb4ebd91a993fea54fc215057bf0da2659056b4bc033873db", ) def _gofumpt_deps(): http_file( name = "com_github_mvdan_gofumpt_linux_amd64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/759c6ab56bfbf62cafb35944aef1e0104a117e0aebfe44816fd79ef4b28521e4", - "https://github.com/mvdan/gofumpt/releases/download/v0.5.0/gofumpt_v0.5.0_linux_amd64", + "https://cdn.confidential.cloud/constellation/cas/sha256/11604bbaf7321abcc2fca2c6a37b7e9198bb1e76e5a86f297c07201e8ab1fda9", + "https://github.com/mvdan/gofumpt/releases/download/v0.8.0/gofumpt_v0.8.0_linux_amd64", ], executable = True, downloaded_file_path = "gofumpt", - sha256 = "759c6ab56bfbf62cafb35944aef1e0104a117e0aebfe44816fd79ef4b28521e4", + sha256 = "11604bbaf7321abcc2fca2c6a37b7e9198bb1e76e5a86f297c07201e8ab1fda9", ) http_file( name = "com_github_mvdan_gofumpt_linux_arm64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/fba20ffd06606c89a500e3cc836408a09e4767e2f117c97724237ae4ecadf82e", - "https://github.com/mvdan/gofumpt/releases/download/v0.5.0/gofumpt_v0.5.0_linux_arm64", + "https://cdn.confidential.cloud/constellation/cas/sha256/787c1d3d4d20e6fe2b0bf06a5a913ac0f50343dbf9a71540724a2b8092a0e6ca", + "https://github.com/mvdan/gofumpt/releases/download/v0.8.0/gofumpt_v0.8.0_linux_arm64", ], executable = True, downloaded_file_path = "gofumpt", - sha256 = "fba20ffd06606c89a500e3cc836408a09e4767e2f117c97724237ae4ecadf82e", + sha256 = "787c1d3d4d20e6fe2b0bf06a5a913ac0f50343dbf9a71540724a2b8092a0e6ca", ) http_file( name = "com_github_mvdan_gofumpt_darwin_amd64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/870f05a23541aad3d20d208a3ea17606169a240f608ac1cf987426198c14b2ed", - "https://github.com/mvdan/gofumpt/releases/download/v0.5.0/gofumpt_v0.5.0_darwin_amd64", + "https://cdn.confidential.cloud/constellation/cas/sha256/0dda6600cf263b703a5ad93e792b06180c36afdee9638617a91dd552f2c6fb3e", + "https://github.com/mvdan/gofumpt/releases/download/v0.8.0/gofumpt_v0.8.0_darwin_amd64", ], executable = True, downloaded_file_path = "gofumpt", - sha256 = "870f05a23541aad3d20d208a3ea17606169a240f608ac1cf987426198c14b2ed", + sha256 = "0dda6600cf263b703a5ad93e792b06180c36afdee9638617a91dd552f2c6fb3e", ) http_file( name = "com_github_mvdan_gofumpt_darwin_arm64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/f2df95d5fad8498ad8eeb0be8abdb8bb8d05e8130b332cb69751dfd090fabac4", - "https://github.com/mvdan/gofumpt/releases/download/v0.5.0/gofumpt_v0.5.0_darwin_arm64", + "https://cdn.confidential.cloud/constellation/cas/sha256/7e66e92b7a67d1d12839ab030fb7ae38e5e2273474af3762e67bc7fe9471fcd9", + "https://github.com/mvdan/gofumpt/releases/download/v0.8.0/gofumpt_v0.8.0_darwin_arm64", ], executable = True, downloaded_file_path = "gofumpt", - sha256 = "f2df95d5fad8498ad8eeb0be8abdb8bb8d05e8130b332cb69751dfd090fabac4", + sha256 = "7e66e92b7a67d1d12839ab030fb7ae38e5e2273474af3762e67bc7fe9471fcd9", ) def _tfsec_deps(): @@ -181,41 +181,41 @@ def _tfsec_deps(): name = "com_github_aquasecurity_tfsec_linux_amd64", build_file_content = """exports_files(["tfsec"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/8b4491bc151e053c2808ee61fc79debc18bc6c778d5ff7ed213b0064760add58", - "https://github.com/aquasecurity/tfsec/releases/download/v1.28.4/tfsec_1.28.4_linux_amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/329ae7f67f2f1813ebe08de498719ea7003c75d3ca24bb0b038369062508008e", + "https://github.com/aquasecurity/tfsec/releases/download/v1.28.14/tfsec_1.28.14_linux_amd64.tar.gz", ], type = "tar.gz", - sha256 = "8b4491bc151e053c2808ee61fc79debc18bc6c778d5ff7ed213b0064760add58", + sha256 = "329ae7f67f2f1813ebe08de498719ea7003c75d3ca24bb0b038369062508008e", ) http_archive( name = "com_github_aquasecurity_tfsec_linux_arm64", build_file_content = """exports_files(["tfsec"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/417ca57cda648e3666b525d6a225d1d182a39031899ef4934ca917f6531277c1", - "https://github.com/aquasecurity/tfsec/releases/download/v1.28.4/tfsec_1.28.4_linux_arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/13dcbd3602027be49ce6cab7e1c24b0a8e833f0143fe327b0a13b87686541ce0", + "https://github.com/aquasecurity/tfsec/releases/download/v1.28.14/tfsec_1.28.14_linux_arm64.tar.gz", ], type = "tar.gz", - sha256 = "417ca57cda648e3666b525d6a225d1d182a39031899ef4934ca917f6531277c1", + sha256 = "13dcbd3602027be49ce6cab7e1c24b0a8e833f0143fe327b0a13b87686541ce0", ) http_archive( name = "com_github_aquasecurity_tfsec_darwin_amd64", build_file_content = """exports_files(["tfsec"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/f67c0c6602721177122d1dd7c14d53ea902410786df361728a166cd3d190945d", - "https://github.com/aquasecurity/tfsec/releases/download/v1.28.4/tfsec_1.28.4_darwin_amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/0aeef31f83d6f44ba9ba5b6cbb954304c772dee73ac704e38896940f94af887a", + "https://github.com/aquasecurity/tfsec/releases/download/v1.28.14/tfsec_1.28.14_darwin_amd64.tar.gz", ], type = "tar.gz", - sha256 = "f67c0c6602721177122d1dd7c14d53ea902410786df361728a166cd3d190945d", + sha256 = "0aeef31f83d6f44ba9ba5b6cbb954304c772dee73ac704e38896940f94af887a", ) http_archive( name = "com_github_aquasecurity_tfsec_darwin_arm64", build_file_content = """exports_files(["tfsec"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/a2f8249f89b16dbddae44a1f5b7a4d9be82b289b048dae57b0dc23f72e5c4f21", - "https://github.com/aquasecurity/tfsec/releases/download/v1.28.4/tfsec_1.28.4_darwin_arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/f39d59a3f9be4eeb3d965657653ad62243103a3d921ce52ca8f907cff45896f5", + "https://github.com/aquasecurity/tfsec/releases/download/v1.28.14/tfsec_1.28.14_darwin_arm64.tar.gz", ], type = "tar.gz", - sha256 = "a2f8249f89b16dbddae44a1f5b7a4d9be82b289b048dae57b0dc23f72e5c4f21", + sha256 = "f39d59a3f9be4eeb3d965657653ad62243103a3d921ce52ca8f907cff45896f5", ) def _golangci_lint_deps(): @@ -223,45 +223,45 @@ def _golangci_lint_deps(): name = "com_github_golangci_golangci_lint_linux_amd64", build_file = "//bazel/toolchains:BUILD.golangci.bazel", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/ca21c961a33be3bc15e4292dc40c98c8dcc5463a7b6768a3afc123761630c09c", - "https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-linux-amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/e55e0eb515936c0fbd178bce504798a9bd2f0b191e5e357768b18fd5415ee541", + "https://github.com/golangci/golangci-lint/releases/download/v2.1.6/golangci-lint-2.1.6-linux-amd64.tar.gz", ], - strip_prefix = "golangci-lint-1.55.2-linux-amd64", + strip_prefix = "golangci-lint-2.1.6-linux-amd64", type = "tar.gz", - sha256 = "ca21c961a33be3bc15e4292dc40c98c8dcc5463a7b6768a3afc123761630c09c", + sha256 = "e55e0eb515936c0fbd178bce504798a9bd2f0b191e5e357768b18fd5415ee541", ) http_archive( name = "com_github_golangci_golangci_lint_linux_arm64", build_file = "//bazel/toolchains:BUILD.golangci.bazel", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/8eb0cee9b1dbf0eaa49871798c7f8a5b35f2960c52d776a5f31eb7d886b92746", - "https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-linux-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/582eb73880f4408d7fb89f12b502d577bd7b0b63d8c681da92bb6b9d934d4363", + "https://github.com/golangci/golangci-lint/releases/download/v2.1.6/golangci-lint-2.1.6-linux-arm64.tar.gz", ], - strip_prefix = "golangci-lint-1.55.2-linux-arm64", + strip_prefix = "golangci-lint-2.1.6-linux-arm64", type = "tar.gz", - sha256 = "8eb0cee9b1dbf0eaa49871798c7f8a5b35f2960c52d776a5f31eb7d886b92746", + sha256 = "582eb73880f4408d7fb89f12b502d577bd7b0b63d8c681da92bb6b9d934d4363", ) http_archive( name = "com_github_golangci_golangci_lint_darwin_amd64", build_file = "//bazel/toolchains:BUILD.golangci.bazel", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/632e96e6d5294fbbe7b2c410a49c8fa01c60712a0af85a567de85bcc1623ea21", - "https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-darwin-amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/e091107c4ca7e283902343ba3a09d14fb56b86e071effd461ce9d67193ef580e", + "https://github.com/golangci/golangci-lint/releases/download/v2.1.6/golangci-lint-2.1.6-darwin-amd64.tar.gz", ], - strip_prefix = "golangci-lint-1.55.2-darwin-amd64", + strip_prefix = "golangci-lint-2.1.6-darwin-amd64", type = "tar.gz", - sha256 = "632e96e6d5294fbbe7b2c410a49c8fa01c60712a0af85a567de85bcc1623ea21", + sha256 = "e091107c4ca7e283902343ba3a09d14fb56b86e071effd461ce9d67193ef580e", ) http_archive( name = "com_github_golangci_golangci_lint_darwin_arm64", build_file = "//bazel/toolchains:BUILD.golangci.bazel", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/234463f059249f82045824afdcdd5db5682d0593052f58f6a3039a0a1c3899f6", - "https://github.com/golangci/golangci-lint/releases/download/v1.55.2/golangci-lint-1.55.2-darwin-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/90783fa092a0f64a4f7b7d419f3da1f53207e300261773babe962957240e9ea6", + "https://github.com/golangci/golangci-lint/releases/download/v2.1.6/golangci-lint-2.1.6-darwin-arm64.tar.gz", ], - strip_prefix = "golangci-lint-1.55.2-darwin-arm64", + strip_prefix = "golangci-lint-2.1.6-darwin-arm64", type = "tar.gz", - sha256 = "234463f059249f82045824afdcdd5db5682d0593052f58f6a3039a0a1c3899f6", + sha256 = "90783fa092a0f64a4f7b7d419f3da1f53207e300261773babe962957240e9ea6", ) def _buf_deps(): @@ -270,44 +270,44 @@ def _buf_deps(): strip_prefix = "buf/bin", build_file_content = """exports_files(["buf"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/0e4294c688e5ee31daa9e01fdcf28946974fa4c0135c59e05dc46947e51cac1d", - "https://github.com/bufbuild/buf/releases/download/v1.28.0/buf-Linux-x86_64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/fa10faf16973f3861992cc2687b651350d70eafd467aea72cf0994556c2a0927", + "https://github.com/bufbuild/buf/releases/download/v1.54.0/buf-Linux-x86_64.tar.gz", ], type = "tar.gz", - sha256 = "0e4294c688e5ee31daa9e01fdcf28946974fa4c0135c59e05dc46947e51cac1d", + sha256 = "fa10faf16973f3861992cc2687b651350d70eafd467aea72cf0994556c2a0927", ) http_archive( name = "com_github_bufbuild_buf_linux_arm64", strip_prefix = "buf/bin", build_file_content = """exports_files(["buf"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/6a7211c3142205458f331b6fda7abd0bd78749af016beb301647aa66708c71a8", - "https://github.com/bufbuild/buf/releases/download/v1.28.0/buf-Linux-aarch64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/f41ef4431858556ece6a77662d6b9317fa4406585998cb3dffb7403b3e86713e", + "https://github.com/bufbuild/buf/releases/download/v1.54.0/buf-Linux-aarch64.tar.gz", ], type = "tar.gz", - sha256 = "6a7211c3142205458f331b6fda7abd0bd78749af016beb301647aa66708c71a8", + sha256 = "f41ef4431858556ece6a77662d6b9317fa4406585998cb3dffb7403b3e86713e", ) http_archive( name = "com_github_bufbuild_buf_darwin_amd64", strip_prefix = "buf/bin", build_file_content = """exports_files(["buf"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/b7391309204c421b83d6696bfd581f6b9d12bdd44c24bdb40a06da77c5a3e577", - "https://github.com/bufbuild/buf/releases/download/v1.28.0/buf-Darwin-x86_64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/22c9836a836b867e49e9d0ef223fd934cbf2690e7400facddb9be07b8809f889", + "https://github.com/bufbuild/buf/releases/download/v1.54.0/buf-Darwin-x86_64.tar.gz", ], type = "tar.gz", - sha256 = "b7391309204c421b83d6696bfd581f6b9d12bdd44c24bdb40a06da77c5a3e577", + sha256 = "22c9836a836b867e49e9d0ef223fd934cbf2690e7400facddb9be07b8809f889", ) http_archive( name = "com_github_bufbuild_buf_darwin_arm64", strip_prefix = "buf/bin", build_file_content = """exports_files(["buf"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/f9b862b445520df8f9b39312f1ecbdcc23b05f9edc10ea03b1d376a7eb4d3511", - "https://github.com/bufbuild/buf/releases/download/v1.28.0/buf-Darwin-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/f01f32a690efab3ef22a1c821aebc0c4bec7ca63faddbf64408d7d614e9d7f92", + "https://github.com/bufbuild/buf/releases/download/v1.54.0/buf-Darwin-arm64.tar.gz", ], type = "tar.gz", - sha256 = "f9b862b445520df8f9b39312f1ecbdcc23b05f9edc10ea03b1d376a7eb4d3511", + sha256 = "f01f32a690efab3ef22a1c821aebc0c4bec7ca63faddbf64408d7d614e9d7f92", ) def _talos_docgen_deps(): @@ -351,46 +351,46 @@ def _talos_docgen_deps(): def _helm_deps(): http_archive( name = "com_github_helm_helm_linux_amd64", - sha256 = "781d826daec584f9d50a01f0f7dadfd25a3312217a14aa2fbb85107b014ac8ca", + sha256 = "a5844ef2c38ef6ddf3b5a8f7d91e7e0e8ebc39a38bb3fc8013d629c1ef29c259", strip_prefix = "linux-amd64", build_file_content = """exports_files(["helm"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/781d826daec584f9d50a01f0f7dadfd25a3312217a14aa2fbb85107b014ac8ca", - "https://get.helm.sh/helm-v3.11.2-linux-amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/a5844ef2c38ef6ddf3b5a8f7d91e7e0e8ebc39a38bb3fc8013d629c1ef29c259", + "https://get.helm.sh/helm-v3.14.4-linux-amd64.tar.gz", ], type = "tar.gz", ) http_archive( name = "com_github_helm_helm_linux_arm64", - sha256 = "0a60baac83c3106017666864e664f52a4e16fbd578ac009f9a85456a9241c5db", + sha256 = "113ccc53b7c57c2aba0cd0aa560b5500841b18b5210d78641acfddc53dac8ab2", strip_prefix = "linux-arm64", build_file_content = """exports_files(["helm"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/0a60baac83c3106017666864e664f52a4e16fbd578ac009f9a85456a9241c5db", - "https://get.helm.sh/helm-v3.11.2-linux-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/113ccc53b7c57c2aba0cd0aa560b5500841b18b5210d78641acfddc53dac8ab2", + "https://get.helm.sh/helm-v3.14.4-linux-arm64.tar.gz", ], type = "tar.gz", ) http_archive( name = "com_github_helm_helm_darwin_amd64", - sha256 = "404938fd2c6eff9e0dab830b0db943fca9e1572cd3d7ee40904705760faa390f", + sha256 = "73434aeac36ad068ce2e5582b8851a286dc628eae16494a26e2ad0b24a7199f9", strip_prefix = "darwin-amd64", build_file_content = """exports_files(["helm"], visibility = ["//visibility:public"])""", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/404938fd2c6eff9e0dab830b0db943fca9e1572cd3d7ee40904705760faa390f", - "https://get.helm.sh/helm-v3.11.2-darwin-amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/73434aeac36ad068ce2e5582b8851a286dc628eae16494a26e2ad0b24a7199f9", + "https://get.helm.sh/helm-v3.14.4-darwin-amd64.tar.gz", ], type = "tar.gz", ) http_archive( name = "com_github_helm_helm_darwin_arm64", - sha256 = "f61a3aa55827de2d8c64a2063fd744b618b443ed063871b79f52069e90813151", + sha256 = "61e9c5455f06b2ad0a1280975bf65892e707adc19d766b0cf4e9006e3b7b4b6c", strip_prefix = "darwin-arm64", build_file_content = """exports_files(["helm"], visibility = ["//visibility:public"])""", type = "tar.gz", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/f61a3aa55827de2d8c64a2063fd744b618b443ed063871b79f52069e90813151", - "https://get.helm.sh/helm-v3.11.2-darwin-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/61e9c5455f06b2ad0a1280975bf65892e707adc19d766b0cf4e9006e3b7b4b6c", + "https://get.helm.sh/helm-v3.14.4-darwin-arm64.tar.gz", ], ) @@ -398,40 +398,40 @@ def _ghh_deps(): http_archive( name = "com_github_katexochen_ghh_linux_amd64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/2f86968d4e11de5d7a4430fd973e41c71c25064d1e71dcc21f8c68174ecde522", - "https://github.com/katexochen/ghh/releases/download/v0.3.1/ghh_0.3.1_linux_amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/32f8a4110d88d80e163212a89a3538a13326494840ac97183d1b20bcc9eac7ba", + "https://github.com/katexochen/ghh/releases/download/v0.3.5/ghh_0.3.5_linux_amd64.tar.gz", ], type = "tar.gz", build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""", - sha256 = "2f86968d4e11de5d7a4430fd973e41c71c25064d1e71dcc21f8c68174ecde522", + sha256 = "32f8a4110d88d80e163212a89a3538a13326494840ac97183d1b20bcc9eac7ba", ) http_archive( name = "com_github_katexochen_ghh_linux_arm64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/e8ae94b120428f0b8d9085a05df8c61ba201ee3811fa1ebf1ef23955b2092635", - "https://github.com/katexochen/ghh/releases/download/v0.3.1/ghh_0.3.1_linux_arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/b43ef1dd2f851eed7c69c87f4f73dd923bd1170cefbde247933d5b398a3319d1", + "https://github.com/katexochen/ghh/releases/download/v0.3.5/ghh_0.3.5_linux_arm64.tar.gz", ], type = "tar.gz", build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""", - sha256 = "e8ae94b120428f0b8d9085a05df8c61ba201ee3811fa1ebf1ef23955b2092635", + sha256 = "b43ef1dd2f851eed7c69c87f4f73dd923bd1170cefbde247933d5b398a3319d1", ) http_archive( name = "com_github_katexochen_ghh_darwin_amd64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/9b324ed7a516b0a6869cb8448ad5ef56bf0d8d5f6a8ad6b63d57e61fec570a2e", - "https://github.com/katexochen/ghh/releases/download/v0.3.1/ghh_0.3.1_darwin_amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/7db9ebb62faf2a31f56e7a8994a971adddec98b3238880ae58b01eb549b8bba3", + "https://github.com/katexochen/ghh/releases/download/v0.3.5/ghh_0.3.5_darwin_amd64.tar.gz", ], type = "tar.gz", build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""", - sha256 = "9b324ed7a516b0a6869cb8448ad5ef56bf0d8d5f6a8ad6b63d57e61fec570a2e", + sha256 = "7db9ebb62faf2a31f56e7a8994a971adddec98b3238880ae58b01eb549b8bba3", ) http_archive( name = "com_github_katexochen_ghh_darwin_arm64", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/bc8252749eb55607a0b32232c72d64742d49d5b2f187bf96c0f31c865e94fcc8", - "https://github.com/katexochen/ghh/releases/download/v0.3.1/ghh_0.3.1_darwin_arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/78a2c8b321893736a2b5de59898794a72b878db9329157f348489c73d4592c6f", + "https://github.com/katexochen/ghh/releases/download/v0.3.5/ghh_0.3.5_darwin_arm64.tar.gz", ], type = "tar.gz", build_file_content = """exports_files(["ghh"], visibility = ["//visibility:public"])""", - sha256 = "bc8252749eb55607a0b32232c72d64742d49d5b2f187bf96c0f31c865e94fcc8", + sha256 = "78a2c8b321893736a2b5de59898794a72b878db9329157f348489c73d4592c6f", ) diff --git a/bazel/toolchains/container_images.bzl b/bazel/toolchains/container_images.bzl index feaa27d06..1c405484f 100644 --- a/bazel/toolchains/container_images.bzl +++ b/bazel/toolchains/container_images.bzl @@ -7,7 +7,7 @@ load("@rules_oci//oci:pull.bzl", "oci_pull") def containter_image_deps(): oci_pull( name = "distroless_static", - digest = "sha256:6706c73aae2afaa8201d63cc3dda48753c09bcd6c300762251065c0f7e602b25", + digest = "sha256:3d0f463de06b7ddff27684ec3bfd0b54a425149d0f8685308b1fdf297b0265e9", image = "gcr.io/distroless/static", platforms = [ "linux/amd64", @@ -16,6 +16,6 @@ def containter_image_deps(): ) oci_pull( name = "libvirtd_base", - digest = "sha256:f5aca956c8d67059725feb4bf8a7d96da71a51efe84288c74a52fcf6855a13bd", + digest = "sha256:f23e0f587860c841adde25b1b4f0d99aa4fbce1c92b01b5b46ab5fa35980a135", image = "ghcr.io/edgelesssys/constellation/libvirtd-base", ) diff --git a/bazel/toolchains/go_module_deps.bzl b/bazel/toolchains/go_module_deps.bzl deleted file mode 100644 index add2ec96e..000000000 --- a/bazel/toolchains/go_module_deps.bzl +++ /dev/null @@ -1,7237 +0,0 @@ -"""Go module dependencies for Bazel. - -Contains the equivalent of go.mod and go.sum files for Bazel. -""" - -load("@bazel_gazelle//:deps.bzl", "go_repository") - -def go_dependencies(): - """Declare Go module dependencies for Bazel.""" - go_repository( - name = "build_buf_gen_go_bufbuild_protovalidate_protocolbuffers_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go", - sum = "h1:tdpHgTbmbvEIARu+bixzmleMi14+3imnpoFXz+Qzjp4=", - version = "v1.31.0-20230802163732-1c33ebd9ecfa.1", - ) - go_repository( - name = "cat_dario_mergo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "dario.cat/mergo", - sum = "h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=", - version = "v1.0.0", - ) - go_repository( - name = "cc_mvdan_editorconfig", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "mvdan.cc/editorconfig", - sum = "h1:XL+7ys6ls/RKrkUNFQvEwIvNHh+JKx8Mj1pUV5wQxQE=", - version = "v0.2.0", - ) - go_repository( - name = "cc_mvdan_unparam", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "mvdan.cc/unparam", - sum = "h1:VuJo4Mt0EVPychre4fNlDWDuE5AjXtPJpRUWqZDQhaI=", - version = "v0.0.0-20230312165513-e84e2d14e3b8", - ) - go_repository( - name = "co_honnef_go_tools", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "honnef.co/go/tools", - sum = "h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=", - version = "v0.0.1-2020.1.4", - ) - go_repository( - name = "com_github_acomagu_bufpipe", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/acomagu/bufpipe", - sum = "h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=", - version = "v1.0.4", - ) - go_repository( - name = "com_github_adalogics_go_fuzz_headers", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/AdaLogics/go-fuzz-headers", - sum = "h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=", - version = "v0.0.0-20230811130428-ced1acdcaa24", - ) - go_repository( - name = "com_github_adamkorcz_go_118_fuzz_build", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/AdamKorcz/go-118-fuzz-build", - sum = "h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA=", - version = "v0.0.0-20230306123547-8075edf89bb0", - ) - go_repository( - name = "com_github_adamkorcz_go_fuzz_headers_1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/AdamKorcz/go-fuzz-headers-1", - sum = "h1:rd389Q26LMy03gG4anandGFC2LW/xvjga5GezeeaxQk=", - version = "v0.0.0-20230618160516-e936619f9f18", - ) - go_repository( - name = "com_github_adrg_xdg", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/adrg/xdg", - sum = "h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_agext_levenshtein", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/agext/levenshtein", - sum = "h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE=", - version = "v1.2.2", - ) - go_repository( - name = "com_github_agnivade_levenshtein", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/agnivade/levenshtein", - sum = "h1:3oJU7J3FGFmyhn8KHjmVaZCN5hxTr7GxgRue+sxIXdQ=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_alcortesm_tgz", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/alcortesm/tgz", - sum = "h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=", - version = "v0.0.0-20161220082320-9c5fe88206d7", - ) - go_repository( - name = "com_github_alecthomas_kingpin_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/alecthomas/kingpin/v2", - sum = "h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU=", - version = "v2.3.2", - ) - go_repository( - name = "com_github_alecthomas_template", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/alecthomas/template", - sum = "h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=", - version = "v0.0.0-20160405071501-a0175ee3bccc", - ) - go_repository( - name = "com_github_alecthomas_units", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/alecthomas/units", - sum = "h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=", - version = "v0.0.0-20211218093645-b94a6e3cc137", - ) - go_repository( - name = "com_github_alessio_shellescape", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/alessio/shellescape", - sum = "h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=", - version = "v1.4.1", - ) - go_repository( - name = "com_github_anatol_vmtest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/anatol/vmtest", - sum = "h1:t4JGeY9oaF5LB4Rdx9e2wARRRPAYt8Ow4eCf5SwO3fA=", - version = "v0.0.0-20220413190228-7a42f1f6d7b8", - ) - go_repository( - name = "com_github_andybalholm_brotli", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/andybalholm/brotli", - sum = "h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=", - version = "v1.0.4", - ) - go_repository( - name = "com_github_anmitsu_go_shlex", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/anmitsu/go-shlex", - sum = "h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=", - version = "v0.0.0-20161002113705-648efa622239", - ) - go_repository( - name = "com_github_antihax_optional", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/antihax/optional", - sum = "h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_antlr_antlr4_runtime_go_antlr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/antlr/antlr4/runtime/Go/antlr", - sum = "h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=", - version = "v1.4.10", - ) - go_repository( - name = "com_github_antlr_antlr4_runtime_go_antlr_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/antlr/antlr4/runtime/Go/antlr/v4", - sum = "h1:goHVqTbFX3AIo0tzGr14pgfAW2ZfPChKO21Z9MGf/gk=", - version = "v4.0.0-20230512164433-5d1fd1a340c9", - ) - go_repository( - name = "com_github_apache_arrow_go_v12", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/apache/arrow/go/v12", - sum = "h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc=", - version = "v12.0.0", - ) - go_repository( - name = "com_github_apache_thrift", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/apache/thrift", - sum = "h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=", - version = "v0.16.0", - ) - go_repository( - name = "com_github_apparentlymart_go_dump", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/apparentlymart/go-dump", - sum = "h1:ZSTrOEhiM5J5RFxEaFvMZVEAM1KvT1YzbEOwB2EAGjA=", - version = "v0.0.0-20180507223929-23540a00eaa3", - ) - go_repository( - name = "com_github_apparentlymart_go_textseg_v12", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/apparentlymart/go-textseg/v12", - sum = "h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0=", - version = "v12.0.0", - ) - go_repository( - name = "com_github_apparentlymart_go_textseg_v13", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/apparentlymart/go-textseg/v13", - sum = "h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=", - version = "v13.0.0", - ) - go_repository( - name = "com_github_apparentlymart_go_textseg_v15", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/apparentlymart/go-textseg/v15", - sum = "h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=", - version = "v15.0.0", - ) - go_repository( - name = "com_github_armon_circbuf", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/armon/circbuf", - sum = "h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA=", - version = "v0.0.0-20150827004946-bbbad097214e", - ) - go_repository( - name = "com_github_armon_go_radix", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/armon/go-radix", - sum = "h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_armon_go_socks5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/armon/go-socks5", - sum = "h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=", - version = "v0.0.0-20160902184237-e75332964ef5", - ) - go_repository( - name = "com_github_asaskevich_govalidator", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/asaskevich/govalidator", - sum = "h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=", - version = "v0.0.0-20230301143203-a9d515a09cc2", - ) - go_repository( - name = "com_github_aws_aws_sdk_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go", - sum = "h1:uL4EV0gQxotQVYegIoBqK079328MOJqgG95daFYSkAM=", - version = "v1.44.297", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2", - sum = "h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=", - version = "v1.18.1", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_aws_protocol_eventstream", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream", - sum = "h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs=", - version = "v1.4.10", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_config", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/config", - sum = "h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=", - version = "v1.18.27", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_credentials", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/credentials", - sum = "h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=", - version = "v1.13.26", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_feature_ec2_imds", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/feature/ec2/imds", - sum = "h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=", - version = "v1.13.4", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_feature_s3_manager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/feature/s3/manager", - sum = "h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA=", - version = "v1.11.71", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_internal_configsources", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/internal/configsources", - sum = "h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=", - version = "v1.1.34", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_internal_endpoints_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2", - sum = "h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=", - version = "v2.4.28", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_internal_ini", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/internal/ini", - sum = "h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=", - version = "v1.3.35", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_internal_v4a", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/internal/v4a", - sum = "h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c=", - version = "v1.0.26", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_autoscaling", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/autoscaling", - sum = "h1:gnNW8xYVF7pKJrIu6WRF2r9NZylc7jLna2O3oPFIii0=", - version = "v1.28.9", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_cloudfront", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/cloudfront", - sum = "h1:loRDtQ0vT0+JCB0hQBCfv95tttEzJ1rqSaTDy5cpy0A=", - version = "v1.26.8", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_cloudwatchlogs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs", - sum = "h1:zMnh9plMceN5DVuG55IjzEwAS3kbeG0GTNzmbnqI/C8=", - version = "v1.21.2", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_ec2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/ec2", - sum = "h1:P4dyjm49F2kKws0FpouBC6fjVImACXKt752+CWa01lM=", - version = "v1.102.0", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2", - sum = "h1:g/Kzed9qNdvz5p7Av3ffavD19eN11deWqlHgR2JuXuw=", - version = "v1.19.13", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_internal_accept_encoding", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding", - sum = "h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA=", - version = "v1.9.11", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_internal_checksum", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/internal/checksum", - sum = "h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ=", - version = "v1.1.29", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_internal_presigned_url", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url", - sum = "h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s=", - version = "v1.9.28", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_internal_s3shared", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/internal/s3shared", - sum = "h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA=", - version = "v1.14.3", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_kms", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/kms", - sum = "h1:jwmtdM1/l1DRNy5jQrrYpsQm8zwetkgeqhAqefDr1yI=", - version = "v1.22.2", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi", - sum = "h1:6AuIiaZ+oRhprPZw2/siZQcaZRvmKipjGbmGI0BSGsA=", - version = "v1.14.14", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_s3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/s3", - sum = "h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4=", - version = "v1.36.0", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_secretsmanager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/secretsmanager", - sum = "h1:eW8zPSh7ZLzb7029xCsIEFbnxLvNHPTt7aWwdKjNJc8=", - version = "v1.19.10", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_sso", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/sso", - sum = "h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=", - version = "v1.12.12", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_ssooidc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/ssooidc", - sum = "h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80=", - version = "v1.14.12", - ) - go_repository( - name = "com_github_aws_aws_sdk_go_v2_service_sts", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/aws-sdk-go-v2/service/sts", - sum = "h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE=", - version = "v1.19.2", - ) - go_repository( - name = "com_github_aws_smithy_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/aws/smithy-go", - sum = "h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=", - version = "v1.13.5", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go", - sum = "h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=", - version = "v68.0.0+incompatible", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_azcore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/azcore", - sum = "h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4=", - version = "v1.6.1", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_azidentity", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/azidentity", - sum = "h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_internal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/internal", - sum = "h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_keyvault_azsecrets", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets", - sum = "h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw=", - version = "v0.12.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_keyvault_internal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal", - sum = "h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw=", - version = "v0.7.1", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_applicationinsights_armapplicationinsights", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights", - sum = "h1:hBrFatNIiVAwDb5GzMLjpkQ6l2/waFSvBWMBWZRH8WI=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5", - sum = "h1:Sg/D8VuUQ+bw+FOYJF+xRKcwizCOP13HL0Se8pWNBzE=", - version = "v5.1.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_internal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal", - sum = "h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=", - version = "v1.1.2", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4", - sum = "h1:pqCyNi/Paz03SbWRmGlb5WBzK14aOXVuSJuOTWzOM5M=", - version = "v4.0.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_resourcemanager_resources_armresources", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources", - sum = "h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_security_keyvault_azkeys", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys", - sum = "h1:4Kynh6Hn2ekyIsBgNQJb3dn1+/MyvzfUJebti2emB/A=", - version = "v0.12.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_security_keyvault_internal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal", - sum = "h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4=", - version = "v0.8.0", - ) - go_repository( - name = "com_github_azure_azure_sdk_for_go_sdk_storage_azblob", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob", - sum = "h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_azure_go_ansiterm", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-ansiterm", - sum = "h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=", - version = "v0.0.0-20210617225240-d185dfc1b5a1", - ) - go_repository( - name = "com_github_azure_go_autorest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest", - sum = "h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=", - version = "v14.2.0+incompatible", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest", - sum = "h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw=", - version = "v0.11.29", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_adal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/adal", - sum = "h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc=", - version = "v0.9.22", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_azure_auth", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/azure/auth", - sum = "h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk=", - version = "v0.5.12", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_azure_cli", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/azure/cli", - sum = "h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc=", - version = "v0.4.6", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_date", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/date", - sum = "h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_mocks", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/mocks", - sum = "h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=", - version = "v0.4.2", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_to", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/to", - sum = "h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_azure_go_autorest_autorest_validation", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/autorest/validation", - sum = "h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac=", - version = "v0.3.1", - ) - go_repository( - name = "com_github_azure_go_autorest_logger", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/logger", - sum = "h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=", - version = "v0.2.1", - ) - go_repository( - name = "com_github_azure_go_autorest_tracing", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Azure/go-autorest/tracing", - sum = "h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=", - version = "v0.6.0", - ) - go_repository( - name = "com_github_azuread_microsoft_authentication_library_for_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/AzureAD/microsoft-authentication-library-for-go", - sum = "h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_bazelbuild_buildtools", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bazelbuild/buildtools", - sum = "h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM=", - version = "v0.0.0-20230317132445-9c3c1fc0106e", - ) - go_repository( - name = "com_github_bazelbuild_rules_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bazelbuild/rules_go", - sum = "h1:Q+vDhH4yzafZ0xHBT0JEVawb+1nDHUXhjvWTqSGCCyU=", - version = "v0.43.0", - ) - go_repository( - name = "com_github_beeker1121_goque", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/beeker1121/goque", - sum = "h1:XbgLdZvVbWsK9HAhAYOp6rksTAdOVYDBQtGSVOLlJrw=", - version = "v1.0.3-0.20191103205551-d618510128af", - ) - go_repository( - name = "com_github_benbjohnson_clock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/benbjohnson/clock", - sum = "h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_beorn7_perks", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/beorn7/perks", - sum = "h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_bgentry_speakeasy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bgentry/speakeasy", - sum = "h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_blang_semver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/blang/semver", - sum = "h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=", - version = "v3.5.1+incompatible", - ) - go_repository( - name = "com_github_blang_semver_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/blang/semver/v4", - sum = "h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=", - version = "v4.0.0", - ) - go_repository( - name = "com_github_bshuster_repo_logrus_logstash_hook", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bshuster-repo/logrus-logstash-hook", - sum = "h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_bufbuild_protocompile", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bufbuild/protocompile", - sum = "h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_bufbuild_protovalidate_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bufbuild/protovalidate-go", - sum = "h1:pJr07sYhliyfj/STAM7hU4J3FKpVeLVKvOBmOTN8j+s=", - version = "v0.2.1", - ) - go_repository( - name = "com_github_bugsnag_bugsnag_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bugsnag/bugsnag-go", - sum = "h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=", - version = "v0.0.0-20141110184014-b1d153021fcd", - ) - go_repository( - name = "com_github_bugsnag_osext", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bugsnag/osext", - sum = "h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=", - version = "v0.0.0-20130617224835-0dd3f918b21b", - ) - go_repository( - name = "com_github_bugsnag_panicwrap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bugsnag/panicwrap", - sum = "h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=", - version = "v0.0.0-20151223152923-e2c28503fcd0", - ) - go_repository( - name = "com_github_burntsushi_toml", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/BurntSushi/toml", - sum = "h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=", - version = "v1.3.2", - ) - go_repository( - name = "com_github_burntsushi_xgb", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/BurntSushi/xgb", - sum = "h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=", - version = "v0.0.0-20160522181843-27f122750802", - ) - go_repository( - name = "com_github_bwesterb_go_ristretto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/bwesterb/go-ristretto", - sum = "h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw=", - version = "v1.2.3", - ) - go_repository( - name = "com_github_cavaliercoder_badio", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cavaliercoder/badio", - sum = "h1:YYUjy5BRwO5zPtfk+aa2gw255FIIoi93zMmuy19o0bc=", - version = "v0.0.0-20160213150051-ce5280129e9e", - ) - go_repository( - name = "com_github_cavaliercoder_go_cpio", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cavaliercoder/go-cpio", - sum = "h1:hHg27A0RSSp2Om9lubZpiMgVbvn39bsUmW9U5h0twqc=", - version = "v0.0.0-20180626203310-925f9528c45e", - ) - go_repository( - name = "com_github_cavaliercoder_go_rpm", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cavaliercoder/go-rpm", - sum = "h1:jP7ki8Tzx9ThnFPLDhBYAhEpI2+jOURnHQNURgsMvnY=", - version = "v0.0.0-20200122174316-8cb9fd9c31a8", - ) - go_repository( - name = "com_github_cenkalti_backoff_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cenkalti/backoff/v3", - sum = "h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=", - version = "v3.2.2", - ) - go_repository( - name = "com_github_cenkalti_backoff_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cenkalti/backoff/v4", - sum = "h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=", - version = "v4.2.1", - ) - go_repository( - name = "com_github_census_instrumentation_opencensus_proto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/census-instrumentation/opencensus-proto", - sum = "h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=", - version = "v0.4.1", - ) - go_repository( - name = "com_github_cespare_xxhash", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cespare/xxhash", - sum = "h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_cespare_xxhash_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cespare/xxhash/v2", - sum = "h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=", - version = "v2.2.0", - ) - go_repository( - name = "com_github_chai2010_gettext_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/chai2010/gettext-go", - sum = "h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_checkpoint_restore_go_criu_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/checkpoint-restore/go-criu/v5", - sum = "h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8=", - version = "v5.3.0", - ) - go_repository( - name = "com_github_chzyer_logex", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/chzyer/logex", - sum = "h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_chzyer_readline", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/chzyer/readline", - sum = "h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI=", - version = "v1.5.0", - ) - go_repository( - name = "com_github_chzyer_test", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/chzyer/test", - sum = "h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ=", - version = "v0.0.0-20210722231415-061457976a23", - ) - go_repository( - name = "com_github_cilium_ebpf", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cilium/ebpf", - sum = "h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4=", - version = "v0.9.1", - ) - go_repository( - name = "com_github_client9_misspell", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/client9/misspell", - sum = "h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=", - version = "v0.3.4", - ) - go_repository( - name = "com_github_cloudflare_circl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cloudflare/circl", - # keep - patches = [ - "//3rdparty/bazel/com_github_cloudflare_circl:math_fp448_BUILD_bazel.patch", - "//3rdparty/bazel/com_github_cloudflare_circl:math_fp25519_BUILD_bazel.patch", - "//3rdparty/bazel/com_github_cloudflare_circl:dh_x448_BUILD_bazel.patch", - "//3rdparty/bazel/com_github_cloudflare_circl:dh_x25519_BUILD_bazel.patch", - ], - sum = "h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=", - version = "v1.3.3", - ) - go_repository( - name = "com_github_cncf_udpa_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cncf/udpa/go", - sum = "h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk=", - version = "v0.0.0-20220112060539-c52dc94e7fbe", - ) - go_repository( - name = "com_github_cncf_xds_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cncf/xds/go", - sum = "h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=", - version = "v0.0.0-20230607035331-e9ce68804cb4", - ) - go_repository( - name = "com_github_codahale_rfc6979", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/codahale/rfc6979", - sum = "h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=", - version = "v0.0.0-20141003034818-6a90f24967eb", - ) - go_repository( - name = "com_github_common_nighthawk_go_figure", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/common-nighthawk/go-figure", - sum = "h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=", - version = "v0.0.0-20210622060536-734e95fb86be", - ) - go_repository( - name = "com_github_container_orchestrated_devices_container_device_interface", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/container-orchestrated-devices/container-device-interface", - sum = "h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8=", - version = "v0.5.4", - ) - go_repository( - name = "com_github_container_storage_interface_spec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/container-storage-interface/spec", - sum = "h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw=", - version = "v1.7.0", - ) - go_repository( - name = "com_github_containerd_aufs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/aufs", - sum = "h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_containerd_btrfs_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/btrfs/v2", - sum = "h1:FN4wsx7KQrYoLXN7uLP0vBV4oVWHOIKDRQ1G2Z0oL5M=", - version = "v2.0.0", - ) - go_repository( - name = "com_github_containerd_cgroups", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/cgroups", - sum = "h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_containerd_cgroups_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/cgroups/v3", - sum = "h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0=", - version = "v3.0.2", - ) - go_repository( - name = "com_github_containerd_console", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/console", - sum = "h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=", - version = "v1.0.3", - ) - go_repository( - name = "com_github_containerd_containerd", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/containerd", - sum = "h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8=", - version = "v1.7.6", - ) - go_repository( - name = "com_github_containerd_continuity", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/continuity", - sum = "h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=", - version = "v0.4.2", - ) - go_repository( - name = "com_github_containerd_fifo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/fifo", - sum = "h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_containerd_go_cni", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/go-cni", - sum = "h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9rU=", - version = "v1.1.9", - ) - go_repository( - name = "com_github_containerd_go_runc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/go-runc", - sum = "h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_containerd_imgcrypt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/imgcrypt", - sum = "h1:WSf9o9EQ0KGHiUx2ESFZ+PKf4nxK9BcvV/nJDX8RkB4=", - version = "v1.1.7", - ) - go_repository( - name = "com_github_containerd_nri", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/nri", - sum = "h1:2ZM4WImye1ypSnE7COjOvPAiLv84kaPILBDvb1tbDK8=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_containerd_stargz_snapshotter_estargz", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/stargz-snapshotter/estargz", - sum = "h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=", - version = "v0.14.3", - ) - go_repository( - name = "com_github_containerd_ttrpc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/ttrpc", - sum = "h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs=", - version = "v1.2.2", - ) - go_repository( - name = "com_github_containerd_typeurl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/typeurl", - sum = "h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_containerd_typeurl_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/typeurl/v2", - sum = "h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4=", - version = "v2.1.1", - ) - go_repository( - name = "com_github_containerd_zfs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containerd/zfs", - sum = "h1:n7OZ7jZumLIqNJqXrEc/paBM840mORnmGdJDmAmJZHM=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_containernetworking_cni", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containernetworking/cni", - sum = "h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=", - version = "v1.1.2", - ) - go_repository( - name = "com_github_containernetworking_plugins", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containernetworking/plugins", - sum = "h1:SWgg3dQG1yzUo4d9iD8cwSVh1VqI+bP7mkPDoSfP9VU=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_containers_ocicrypt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/containers/ocicrypt", - sum = "h1:uoG52u2e91RE4UqmBICZY8dNshgfvkdl3BW6jnxiFaI=", - version = "v1.1.6", - ) - go_repository( - name = "com_github_coredns_caddy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/coredns/caddy", - sum = "h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_coredns_corefile_migration", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/coredns/corefile-migration", - sum = "h1:MdOkT6F3ehju/n9tgxlGct8XAajOX2vN+wG7To4BWSI=", - version = "v1.0.20", - ) - go_repository( - name = "com_github_coreos_go_oidc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/coreos/go-oidc", - sum = "h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk=", - version = "v2.2.1+incompatible", - ) - go_repository( - name = "com_github_coreos_go_oidc_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/coreos/go-oidc/v3", - sum = "h1:AKVxfYw1Gmkn/w96z0DbT/B/xFnzTd3MkZvWLjF4n/o=", - version = "v3.6.0", - ) - go_repository( - name = "com_github_coreos_go_semver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/coreos/go-semver", - sum = "h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=", - version = "v0.3.1", - ) - go_repository( - name = "com_github_coreos_go_systemd_v22", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/coreos/go-systemd/v22", - sum = "h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=", - version = "v22.5.0", - ) - go_repository( - name = "com_github_cosi_project_runtime", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cosi-project/runtime", - sum = "h1:f8++A7HUu7pQv9G3IhQworfA4TFLdzGWl3W+jLQF3Oo=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_cpuguy83_go_md2man_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cpuguy83/go-md2man/v2", - sum = "h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=", - version = "v2.0.2", - ) - go_repository( - name = "com_github_creack_pty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/creack/pty", - sum = "h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=", - version = "v1.1.18", - ) - go_repository( - name = "com_github_cyberphone_json_canonicalization", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cyberphone/json-canonicalization", - sum = "h1:vU+EP9ZuFUCYE0NYLwTSob+3LNEJATzNfP/DC7SWGWI=", - version = "v0.0.0-20220623050100-57a0ce2678a7", - ) - go_repository( - name = "com_github_cyphar_filepath_securejoin", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/cyphar/filepath-securejoin", - sum = "h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=", - version = "v0.2.4", - ) - go_repository( - name = "com_github_danieljoos_wincred", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/danieljoos/wincred", - sum = "h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=", - version = "v1.1.2", - ) - go_repository( - name = "com_github_data_dog_go_sqlmock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/DATA-DOG/go-sqlmock", - sum = "h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=", - version = "v1.5.0", - ) - go_repository( - name = "com_github_davecgh_go_spew", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/davecgh/go-spew", - sum = "h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_daviddengcn_go_colortext", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/daviddengcn/go-colortext", - sum = "h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_decred_dcrd_dcrec_secp256k1_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/decred/dcrd/dcrec/secp256k1/v4", - sum = "h1:1iy2qD6JEhHKKhUOA9IWs7mjco7lnw2qx8FsRI2wirE=", - version = "v4.0.0-20210816181553-5444fa50b93d", - ) - go_repository( - name = "com_github_denisenkom_go_mssqldb", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/denisenkom/go-mssqldb", - sum = "h1:RSohk2RsiZqLZ0zCjtfn3S4Gp4exhpBWHyQ7D0yGjAk=", - version = "v0.9.0", - ) - go_repository( - name = "com_github_dgryski_go_rendezvous", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/dgryski/go-rendezvous", - sum = "h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=", - version = "v0.0.0-20200823014737-9f7001d12a5f", - ) - go_repository( - name = "com_github_dimchansky_utfbom", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/dimchansky/utfbom", - sum = "h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_distribution_distribution_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/distribution/distribution/v3", - sum = "h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=", - version = "v3.0.0-20221208165359-362910506bc2", - ) - go_repository( - name = "com_github_dnaeon_go_vcr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/dnaeon/go-vcr", - sum = "h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_docker_cli", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/cli", - sum = "h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY=", - version = "v24.0.6+incompatible", - ) - go_repository( - name = "com_github_docker_distribution", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/distribution", - sum = "h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=", - version = "v2.8.2+incompatible", - ) - go_repository( - name = "com_github_docker_docker", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/docker", - sum = "h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=", - version = "v24.0.7+incompatible", - ) - go_repository( - name = "com_github_docker_docker_credential_helpers", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/docker-credential-helpers", - sum = "h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=", - version = "v0.7.0", - ) - go_repository( - name = "com_github_docker_go_connections", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/go-connections", - sum = "h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_docker_go_events", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/go-events", - sum = "h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=", - version = "v0.0.0-20190806004212-e31b211e4f1c", - ) - go_repository( - name = "com_github_docker_go_metrics", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/go-metrics", - sum = "h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=", - version = "v0.0.1", - ) - go_repository( - name = "com_github_docker_go_units", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/go-units", - sum = "h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_docker_libtrust", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/docker/libtrust", - sum = "h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=", - version = "v0.0.0-20150114040149-fa567046d9b1", - ) - go_repository( - name = "com_github_dustin_go_humanize", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/dustin/go-humanize", - sum = "h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_edgelesssys_go_azguestattestation", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/edgelesssys/go-azguestattestation", - sum = "h1:1iKB7b+i7svWC0aKXwggi+kHf0K57g8r9hN4VOpJYYg=", - version = "v0.0.0-20230707101700-a683be600fcf", - ) - go_repository( - name = "com_github_edgelesssys_go_tdx_qpl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/edgelesssys/go-tdx-qpl", - sum = "h1:Q2TI34V/NCLGQQkdc0/KmPx/7ix9YnGDUQDT+gqvDw0=", - version = "v0.0.0-20230530085549-fd2878a4dead", - ) - go_repository( - name = "com_github_eggsampler_acme_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/eggsampler/acme/v3", - sum = "h1:5M7vwYRy65iPpCFHZ01RyWXmYT8e8MlcWn/9BUUB7Ro=", - version = "v3.3.0", - ) - go_repository( - name = "com_github_emicklei_go_restful_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/emicklei/go-restful/v3", - sum = "h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ=", - version = "v3.10.1", - ) - go_repository( - name = "com_github_emirpasic_gods", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/emirpasic/gods", - sum = "h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=", - version = "v1.18.1", - ) - go_repository( - name = "com_github_envoyproxy_go_control_plane", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/envoyproxy/go-control-plane", - sum = "h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM=", - version = "v0.11.1", - ) - go_repository( - name = "com_github_envoyproxy_protoc_gen_validate", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/envoyproxy/protoc-gen-validate", - sum = "h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_euank_go_kmsg_parser", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/euank/go-kmsg-parser", - sum = "h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY=", - version = "v2.0.0+incompatible", - ) - go_repository( - name = "com_github_evanphx_json_patch", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/evanphx/json-patch", - sum = "h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=", - version = "v5.6.0+incompatible", - ) - go_repository( - name = "com_github_evanphx_json_patch_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/evanphx/json-patch/v5", - sum = "h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=", - version = "v5.6.0", - ) - go_repository( - name = "com_github_exponent_io_jsonpath", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/exponent-io/jsonpath", - sum = "h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=", - version = "v0.0.0-20151013193312-d6023ce2651d", - ) - go_repository( - name = "com_github_facebookgo_clock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/facebookgo/clock", - sum = "h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=", - version = "v0.0.0-20150410010913-600d898af40a", - ) - go_repository( - name = "com_github_facebookgo_limitgroup", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/facebookgo/limitgroup", - sum = "h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg=", - version = "v0.0.0-20150612190941-6abd8d71ec01", - ) - go_repository( - name = "com_github_facebookgo_muster", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/facebookgo/muster", - sum = "h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8=", - version = "v0.0.0-20150708232844-fd3d7953fd52", - ) - go_repository( - name = "com_github_fatih_camelcase", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/fatih/camelcase", - sum = "h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_fatih_color", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/fatih/color", - sum = "h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=", - version = "v1.15.0", - ) - go_repository( - name = "com_github_favadi_protoc_go_inject_tag", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/favadi/protoc-go-inject-tag", - sum = "h1:K3KXxbgRw5WT4f43LbglARGz/8jVsDOS7uMjG4oNvXY=", - version = "v1.4.0", - ) - go_repository( - name = "com_github_felixge_httpsnoop", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/felixge/httpsnoop", - sum = "h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=", - version = "v1.0.3", - ) - go_repository( - name = "com_github_flynn_go_docopt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/flynn/go-docopt", - sum = "h1:Ss/B3/5wWRh8+emnK0++g5zQzwDTi30W10pKxKc4JXI=", - version = "v0.0.0-20140912013429-f6dd2ebbb31e", - ) - go_repository( - name = "com_github_flynn_go_shlex", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/flynn/go-shlex", - sum = "h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=", - version = "v0.0.0-20150515145356-3f9db97f8568", - ) - go_repository( - name = "com_github_form3tech_oss_jwt_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/form3tech-oss/jwt-go", - sum = "h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8=", - version = "v3.2.5+incompatible", - ) - go_repository( - name = "com_github_foxboron_go_uefi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/foxboron/go-uefi", - sum = "h1:SJMQFT74bCrP+kQ24oWhmuyPFHDTavrd3JMIe//2NhU=", - version = "v0.0.0-20230808201820-18b9ba9cd4c3", - ) - go_repository( - name = "com_github_foxcpp_go_mockdns", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/foxcpp/go-mockdns", - sum = "h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_frankban_quicktest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/frankban/quicktest", - sum = "h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA=", - version = "v1.14.5", - ) - go_repository( - name = "com_github_fsnotify_fsnotify", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/fsnotify/fsnotify", - sum = "h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=", - version = "v1.7.0", - ) - go_repository( - name = "com_github_fullstorydev_grpcurl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/fullstorydev/grpcurl", - sum = "h1:xJWosq3BQovQ4QrdPO72OrPiWuGgEsxY8ldYsJbPrqI=", - version = "v1.8.7", - ) - go_repository( - name = "com_github_fvbommel_sortorder", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/fvbommel/sortorder", - sum = "h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_fxamacker_cbor_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/fxamacker/cbor/v2", - sum = "h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88=", - version = "v2.4.0", - ) - go_repository( - name = "com_github_gabriel_vasile_mimetype", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gabriel-vasile/mimetype", - sum = "h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=", - version = "v1.4.2", - ) - go_repository( - name = "com_github_gertd_go_pluralize", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gertd/go-pluralize", - sum = "h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA=", - version = "v0.2.1", - ) - go_repository( - name = "com_github_ghodss_yaml", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ghodss/yaml", - sum = "h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_gliderlabs_ssh", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gliderlabs/ssh", - sum = "h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=", - version = "v0.2.2", - ) - go_repository( - name = "com_github_go_chi_chi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-chi/chi", - sum = "h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=", - version = "v4.1.2+incompatible", - ) - go_repository( - name = "com_github_go_errors_errors", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-errors/errors", - sum = "h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=", - version = "v1.4.2", - ) - go_repository( - name = "com_github_go_git_gcfg", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-git/gcfg", - sum = "h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=", - version = "v1.5.1-0.20230307220236-3a3c6141e376", - ) - go_repository( - name = "com_github_go_git_go_billy_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-git/go-billy/v5", - sum = "h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=", - version = "v5.5.0", - ) - go_repository( - name = "com_github_go_git_go_git_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-git/go-git/v5", - sum = "h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=", - version = "v5.9.0", - ) - go_repository( - name = "com_github_go_gl_glfw", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-gl/glfw", - sum = "h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0=", - version = "v0.0.0-20190409004039-e6da0acd62b1", - ) - go_repository( - name = "com_github_go_gl_glfw_v3_3_glfw", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-gl/glfw/v3.3/glfw", - sum = "h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I=", - version = "v0.0.0-20200222043503-6f7a984d4dc4", - ) - go_repository( - name = "com_github_go_gorp_gorp_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-gorp/gorp/v3", - sum = "h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs=", - version = "v3.1.0", - ) - go_repository( - name = "com_github_go_jose_go_jose_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-jose/go-jose/v3", - sum = "h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=", - version = "v3.0.0", - ) - go_repository( - name = "com_github_go_kit_kit", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-kit/kit", - sum = "h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=", - version = "v0.8.0", - ) - go_repository( - name = "com_github_go_kit_log", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-kit/log", - sum = "h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=", - version = "v0.2.1", - ) - go_repository( - name = "com_github_go_logfmt_logfmt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-logfmt/logfmt", - sum = "h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=", - version = "v0.5.1", - ) - go_repository( - name = "com_github_go_logr_logr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-logr/logr", - sum = "h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=", - version = "v1.2.4", - ) - go_repository( - name = "com_github_go_logr_stdr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-logr/stdr", - sum = "h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=", - version = "v1.2.2", - ) - go_repository( - name = "com_github_go_logr_zapr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-logr/zapr", - sum = "h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=", - version = "v1.2.4", - ) - go_repository( - name = "com_github_go_openapi_analysis", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/analysis", - sum = "h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc=", - version = "v0.21.4", - ) - go_repository( - name = "com_github_go_openapi_errors", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/errors", - sum = "h1:unTcVm6PispJsMECE3zWgvG4xTiKda1LIR5rCRWLG6M=", - version = "v0.20.4", - ) - go_repository( - name = "com_github_go_openapi_jsonpointer", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/jsonpointer", - sum = "h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=", - version = "v0.19.6", - ) - go_repository( - name = "com_github_go_openapi_jsonreference", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/jsonreference", - sum = "h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=", - version = "v0.20.2", - ) - go_repository( - name = "com_github_go_openapi_loads", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/loads", - sum = "h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro=", - version = "v0.21.2", - ) - go_repository( - name = "com_github_go_openapi_runtime", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/runtime", - sum = "h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc=", - version = "v0.26.0", - ) - go_repository( - name = "com_github_go_openapi_spec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/spec", - sum = "h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8=", - version = "v0.20.9", - ) - go_repository( - name = "com_github_go_openapi_strfmt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/strfmt", - sum = "h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k=", - version = "v0.21.7", - ) - go_repository( - name = "com_github_go_openapi_swag", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/swag", - sum = "h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=", - version = "v0.22.4", - ) - go_repository( - name = "com_github_go_openapi_validate", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-openapi/validate", - sum = "h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU=", - version = "v0.22.1", - ) - go_repository( - name = "com_github_go_playground_assert_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-playground/assert/v2", - sum = "h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=", - version = "v2.2.0", - ) - go_repository( - name = "com_github_go_playground_locales", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-playground/locales", - sum = "h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=", - version = "v0.14.1", - ) - go_repository( - name = "com_github_go_playground_universal_translator", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-playground/universal-translator", - sum = "h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=", - version = "v0.18.1", - ) - go_repository( - name = "com_github_go_playground_validator_v10", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-playground/validator/v10", - sum = "h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k=", - version = "v10.14.1", - ) - go_repository( - name = "com_github_go_redis_redis_v8", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-redis/redis/v8", - sum = "h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=", - version = "v8.11.5", - ) - go_repository( - name = "com_github_go_redis_redismock_v9", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-redis/redismock/v9", - sum = "h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0=", - version = "v9.0.3", - ) - go_repository( - name = "com_github_go_rod_rod", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-rod/rod", - sum = "h1:oLiKZW721CCMwA5g7977cWfcAKQ+FuosP47Zf1QiDrA=", - version = "v0.113.3", - ) - go_repository( - name = "com_github_go_sql_driver_mysql", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-sql-driver/mysql", - sum = "h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_go_stack_stack", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-stack/stack", - sum = "h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=", - version = "v1.8.0", - ) - go_repository( - name = "com_github_go_task_slim_sprig", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-task/slim-sprig", - sum = "h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=", - version = "v0.0.0-20230315185526-52ccab3ef572", - ) - go_repository( - name = "com_github_go_test_deep", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-test/deep", - sum = "h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_gobuffalo_attrs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/attrs", - sum = "h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4=", - version = "v0.0.0-20190224210810-a9411de4debd", - ) - go_repository( - name = "com_github_gobuffalo_depgen", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/depgen", - sum = "h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_gobuffalo_envy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/envy", - sum = "h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU=", - version = "v1.7.0", - ) - go_repository( - name = "com_github_gobuffalo_flect", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/flect", - sum = "h1:3GQ53z7E3o00C/yy7Ko8VXqQXoJGLkrTQCLTF1EjoXU=", - version = "v0.1.3", - ) - go_repository( - name = "com_github_gobuffalo_genny", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/genny", - sum = "h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0=", - version = "v0.1.1", - ) - go_repository( - name = "com_github_gobuffalo_gitgen", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/gitgen", - sum = "h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8=", - version = "v0.0.0-20190315122116-cc086187d211", - ) - go_repository( - name = "com_github_gobuffalo_gogen", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/gogen", - sum = "h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI=", - version = "v0.1.1", - ) - go_repository( - name = "com_github_gobuffalo_logger", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/logger", - sum = "h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU=", - version = "v1.0.6", - ) - go_repository( - name = "com_github_gobuffalo_mapi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/mapi", - sum = "h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_gobuffalo_packd", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/packd", - sum = "h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_gobuffalo_packr_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/packr/v2", - sum = "h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY=", - version = "v2.8.3", - ) - go_repository( - name = "com_github_gobuffalo_syncx", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobuffalo/syncx", - sum = "h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4=", - version = "v0.0.0-20190224160051-33c29581e754", - ) - go_repository( - name = "com_github_gobwas_glob", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gobwas/glob", - sum = "h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=", - version = "v0.2.3", - ) - go_repository( - name = "com_github_goccy_go_json", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/goccy/go-json", - sum = "h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=", - version = "v0.9.11", - ) - go_repository( - name = "com_github_godbus_dbus_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/godbus/dbus/v5", - sum = "h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=", - version = "v5.1.0", - ) - go_repository( - name = "com_github_godror_godror", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/godror/godror", - sum = "h1:uxGAD7UdnNGjX5gf4NnEIGw0JAPTIFiqAyRBZTPKwXs=", - version = "v0.24.2", - ) - go_repository( - name = "com_github_gofrs_flock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gofrs/flock", - sum = "h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=", - version = "v0.8.1", - ) - go_repository( - name = "com_github_gofrs_uuid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gofrs/uuid", - sum = "h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=", - version = "v4.2.0+incompatible", - ) - go_repository( - name = "com_github_gogo_protobuf", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gogo/protobuf", - sum = "h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=", - version = "v1.3.2", - ) - go_repository( - name = "com_github_golang_glog", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang/glog", - sum = "h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=", - version = "v1.1.2", - ) - go_repository( - name = "com_github_golang_groupcache", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang/groupcache", - sum = "h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=", - version = "v0.0.0-20210331224755-41bb18bfe9da", - ) - go_repository( - name = "com_github_golang_jwt_jwt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang-jwt/jwt", - sum = "h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=", - version = "v3.2.1+incompatible", - ) - go_repository( - name = "com_github_golang_jwt_jwt_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang-jwt/jwt/v4", - sum = "h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=", - version = "v4.5.0", - ) - go_repository( - name = "com_github_golang_jwt_jwt_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang-jwt/jwt/v5", - sum = "h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=", - version = "v5.0.0", - ) - go_repository( - name = "com_github_golang_mock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang/mock", - sum = "h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=", - version = "v1.6.0", - ) - go_repository( - name = "com_github_golang_protobuf", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang/protobuf", - sum = "h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=", - version = "v1.5.3", - ) - go_repository( - name = "com_github_golang_snappy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang/snappy", - sum = "h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=", - version = "v0.0.4", - ) - go_repository( - name = "com_github_golang_sql_civil", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang-sql/civil", - sum = "h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=", - version = "v0.0.0-20190719163853-cb61b32ac6fe", - ) - go_repository( - name = "com_github_gomodule_redigo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gomodule/redigo", - sum = "h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=", - version = "v1.8.2", - ) - go_repository( - name = "com_github_google_btree", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/btree", - sum = "h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=", - version = "v1.1.2", - ) - go_repository( - name = "com_github_google_cadvisor", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/cadvisor", - sum = "h1:lOS3Yprk88AIUi260eKYmTC4pKWAFkXV6Xev5jfCIP8=", - version = "v0.47.2", - ) - go_repository( - name = "com_github_google_cel_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/cel-go", - sum = "h1:s2151PDGy/eqpCI80/8dl4VL3xTkqI/YubXLXCFw0mw=", - version = "v0.17.1", - ) - go_repository( - name = "com_github_google_certificate_transparency_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/certificate-transparency-go", - sum = "h1:hCyXHDbtqlr/lMXU0D4WgbalXL0Zk4dSWWMbPV8VrqY=", - version = "v1.1.4", - ) - go_repository( - name = "com_github_google_flatbuffers", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/flatbuffers", - sum = "h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=", - version = "v2.0.8+incompatible", - ) - go_repository( - name = "com_github_google_gnostic", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/gnostic", - sum = "h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=", - version = "v0.5.7-v3refs", - ) - go_repository( - name = "com_github_google_gnostic_models", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/gnostic-models", - sum = "h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=", - version = "v0.6.8", - ) - go_repository( - name = "com_github_google_go_attestation", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-attestation", - sum = "h1:jXtAWT2sw2Yu8mYU0BC7FDidR+ngxFPSE+pl6IUu3/0=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_google_go_cmdtest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-cmdtest", - sum = "h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=", - version = "v0.4.1-0.20220921163831-55ab3332a786", - ) - go_repository( - name = "com_github_google_go_cmp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-cmp", - sum = "h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=", - version = "v0.6.0", - ) - go_repository( - name = "com_github_google_go_containerregistry", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-containerregistry", - sum = "h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=", - version = "v0.15.2", - ) - go_repository( - name = "com_github_google_go_licenses", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-licenses", - sum = "h1:MM+VCXf0slYkpWO0mECvdYDVCxZXIQNal5wqUIXEZ/A=", - version = "v1.6.0", - ) - go_repository( - name = "com_github_google_go_pkcs11", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-pkcs11", - sum = "h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk=", - version = "v0.2.1-0.20230907215043-c6f79328ddf9", - ) - go_repository( - name = "com_github_google_go_replayers_httpreplay", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-replayers/httpreplay", - sum = "h1:H91sIMlt1NZzN7R+/ASswyouLJfW0WLW7fhyUFvDEkY=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_google_go_sev_guest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-sev-guest", - replace = "github.com/google/go-sev-guest", - sum = "h1:6o4Z/vQqNUH+cEagfx1Ez5ElK70iZulEXZwmLnRo44I=", - version = "v0.0.0-20230928233922-2dcbba0a4b9d", - ) - go_repository( - name = "com_github_google_go_tdx_guest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-tdx-guest", - sum = "h1:lRlUusuieEuqljjihCXb+Mr73VNitOYPJYWXzJKtBWs=", - version = "v0.2.3-0.20231011100059-4cf02bed9d33", - ) - go_repository( - name = "com_github_google_go_tpm", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-tpm", - replace = "github.com/thomasten/go-tpm", - sum = "h1:840nUyrM9df2aLuzWuIkYx/DrUbX4KQZO6B9LD45aWo=", - version = "v0.0.0-20230629092004-f43f8e2a59eb", - ) - go_repository( - name = "com_github_google_go_tpm_tools", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-tpm-tools", - # keep - patches = [ - "//3rdparty/bazel/com_github_google_go_tpm_tools:com_github_google_go_tpm_tools.patch", - "//3rdparty/bazel/com_github_google_go_tpm_tools:ms_tpm_20_ref.patch", - "//3rdparty/bazel/com_github_google_go_tpm_tools:include.patch", - ], - sum = "h1:iyaCPKt2N5Rd0yz0G8ANa022SgCNZkMpp+db6QELtvI=", - version = "v0.4.2", - ) - go_repository( - name = "com_github_google_go_tspi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/go-tspi", - sum = "h1:ADtq8RKfP+jrTyIWIZDIYcKOMecRqNJFOew2IT0Inus=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_google_gofuzz", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/gofuzz", - sum = "h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_google_licenseclassifier", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/licenseclassifier", - sum = "h1:TJsAqW6zLRMDTyGmc9TPosfn9OyVlHs8Hrn3pY6ONSY=", - version = "v0.0.0-20210722185704-3043a050f148", - ) - go_repository( - name = "com_github_google_logger", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/logger", - sum = "h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_google_martian", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/martian", - sum = "h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=", - version = "v2.1.0+incompatible", - ) - go_repository( - name = "com_github_google_martian_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/martian/v3", - sum = "h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=", - version = "v3.3.2", - ) - go_repository( - name = "com_github_google_pprof", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/pprof", - sum = "h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY=", - version = "v0.0.0-20221103000818-d260c55eee4c", - ) - go_repository( - name = "com_github_google_renameio", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/renameio", - sum = "h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_google_renameio_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/renameio/v2", - sum = "h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=", - version = "v2.0.0", - ) - go_repository( - name = "com_github_google_rpmpack", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/rpmpack", - sum = "h1:Fv9Ni1vIq9+Gv4Sm0Xq+NnPYcnsMbdNhJ4Cu4rkbPBM=", - version = "v0.0.0-20210518075352-dc539ef4f2ea", - ) - go_repository( - name = "com_github_google_s2a_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/s2a-go", - sum = "h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=", - version = "v0.1.7", - ) - go_repository( - name = "com_github_google_shlex", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/shlex", - sum = "h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=", - version = "v0.0.0-20191202100458-e7afc7fbc510", - ) - go_repository( - name = "com_github_google_trillian", - build_file_generation = "on", - build_file_name = "", # keep - build_file_proto_mode = "disable_global", - importpath = "github.com/google/trillian", - sum = "h1:roGP6G8aaAch7vP08+oitPkvmZzxjTfIkguozqJ04Ok=", - version = "v1.5.2", - ) - go_repository( - name = "com_github_google_uuid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/uuid", - sum = "h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=", - version = "v1.4.0", - ) - go_repository( - name = "com_github_google_wire", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/google/wire", - sum = "h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_googleapis_enterprise_certificate_proxy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/googleapis/enterprise-certificate-proxy", - sum = "h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=", - version = "v0.3.1", - ) - go_repository( - name = "com_github_googleapis_gax_go_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/googleapis/gax-go/v2", - sum = "h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=", - version = "v2.12.0", - ) - go_repository( - name = "com_github_googleapis_go_type_adapters", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/googleapis/go-type-adapters", - sum = "h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_googleapis_google_cloud_go_testing", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/googleapis/google-cloud-go-testing", - sum = "h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4=", - version = "v0.0.0-20200911160855-bcd43fbb19e8", - ) - go_repository( - name = "com_github_googlecloudplatform_k8s_cloud_provider", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/GoogleCloudPlatform/k8s-cloud-provider", - sum = "h1:Heo1J/ttaQFgGJSVnCZquy3e5eH5j1nqxBuomztB3P0=", - version = "v1.18.1-0.20220218231025-f11817397a1b", - ) - go_repository( - name = "com_github_gophercloud_gophercloud", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gophercloud/gophercloud", - sum = "h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo=", - version = "v1.5.0", - ) - go_repository( - name = "com_github_gophercloud_utils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gophercloud/utils", - sum = "h1:sH7xkTfYzxIEgzq1tDHIMKRh1vThOEOGNsettdEeLbE=", - version = "v0.0.0-20231010081019-80377eca5d56", - ) - go_repository( - name = "com_github_gorilla_handlers", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gorilla/handlers", - sum = "h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=", - version = "v1.5.1", - ) - go_repository( - name = "com_github_gorilla_mux", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gorilla/mux", - sum = "h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=", - version = "v1.8.0", - ) - go_repository( - name = "com_github_gorilla_websocket", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gorilla/websocket", - sum = "h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=", - version = "v1.4.2", - ) - go_repository( - name = "com_github_gosuri_uitable", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gosuri/uitable", - sum = "h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY=", - version = "v0.0.4", - ) - go_repository( - name = "com_github_gregjones_httpcache", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/gregjones/httpcache", - sum = "h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=", - version = "v0.0.0-20190611155906-901d90724c79", - ) - go_repository( - name = "com_github_grpc_ecosystem_go_grpc_middleware", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/grpc-ecosystem/go-grpc-middleware", - sum = "h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_grpc_ecosystem_go_grpc_middleware_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/grpc-ecosystem/go-grpc-middleware/v2", - sum = "h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA=", - version = "v2.0.0", - ) - go_repository( - name = "com_github_grpc_ecosystem_go_grpc_prometheus", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/grpc-ecosystem/go-grpc-prometheus", - sum = "h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_grpc_ecosystem_grpc_gateway", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/grpc-ecosystem/grpc-gateway", - sum = "h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=", - version = "v1.16.0", - ) - go_repository( - name = "com_github_grpc_ecosystem_grpc_gateway_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/grpc-ecosystem/grpc-gateway/v2", - sum = "h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE=", - version = "v2.15.2", - ) - go_repository( - name = "com_github_hashicorp_errwrap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/errwrap", - sum = "h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_hashicorp_go_checkpoint", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-checkpoint", - sum = "h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_hashicorp_go_cleanhttp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-cleanhttp", - sum = "h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=", - version = "v0.5.2", - ) - go_repository( - name = "com_github_hashicorp_go_cty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-cty", - sum = "h1:1/D3zfFHttUKaCaGKZ/dR2roBXv0vKbSCnssIldfQdI=", - version = "v1.4.1-0.20200414143053-d3edf31b6320", - ) - go_repository( - name = "com_github_hashicorp_go_hclog", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-hclog", - sum = "h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=", - version = "v1.5.0", - ) - go_repository( - name = "com_github_hashicorp_go_kms_wrapping_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-kms-wrapping/v2", - sum = "h1:A51EguZ576URdtcQ0l8mT/tOD948oAtmP1soqIHIFfI=", - version = "v2.0.10", - ) - go_repository( - name = "com_github_hashicorp_go_kms_wrapping_wrappers_awskms_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2", - sum = "h1:E3eEWpkofgPNrYyYznfS1+drq4/jFcqHQVNcL7WhUCo=", - version = "v2.0.7", - ) - go_repository( - name = "com_github_hashicorp_go_kms_wrapping_wrappers_azurekeyvault_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2", - sum = "h1:X27JWuPW6Gmi2l7NMm0pvnp7z7hhtns2TeIOQU93mqI=", - version = "v2.0.7", - ) - go_repository( - name = "com_github_hashicorp_go_kms_wrapping_wrappers_gcpckms_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2", - sum = "h1:16I8OqBEuxZIowwn3jiLvhlx+z+ia4dJc9stvz0yUBU=", - version = "v2.0.8", - ) - go_repository( - name = "com_github_hashicorp_go_multierror", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-multierror", - sum = "h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_hashicorp_go_plugin", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-plugin", - sum = "h1:aWv8eimFqWlsEiMrYZdPYl+FdHaBJSN4AWwGWfT1G2Y=", - version = "v1.5.2", - ) - go_repository( - name = "com_github_hashicorp_go_retryablehttp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-retryablehttp", - sum = "h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=", - version = "v0.7.4", - ) - go_repository( - name = "com_github_hashicorp_go_rootcerts", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-rootcerts", - sum = "h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_hashicorp_go_secure_stdlib_awsutil", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-secure-stdlib/awsutil", - sum = "h1:kWg2vyKl7BRXrNxYziqDJ55n+vtOQ1QsGORjzoeB+uM=", - version = "v0.2.2", - ) - go_repository( - name = "com_github_hashicorp_go_secure_stdlib_parseutil", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-secure-stdlib/parseutil", - sum = "h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs=", - version = "v0.1.7", - ) - go_repository( - name = "com_github_hashicorp_go_secure_stdlib_strutil", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-secure-stdlib/strutil", - sum = "h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=", - version = "v0.1.2", - ) - go_repository( - name = "com_github_hashicorp_go_sockaddr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-sockaddr", - sum = "h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_hashicorp_go_uuid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-uuid", - sum = "h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=", - version = "v1.0.3", - ) - go_repository( - name = "com_github_hashicorp_go_version", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-version", - sum = "h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=", - version = "v1.6.0", - ) - go_repository( - name = "com_github_hashicorp_golang_lru", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/golang-lru", - sum = "h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=", - version = "v0.5.4", - ) - go_repository( - name = "com_github_hashicorp_hc_install", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/hc-install", - sum = "h1:IGxShH7AVhPaSuSJpKtVi/EFORNjO+OYVJJrAtGG2mY=", - version = "v0.6.1", - ) - go_repository( - name = "com_github_hashicorp_hcl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/hcl", - sum = "h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_hashicorp_hcl_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/hcl/v2", - sum = "h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI=", - version = "v2.19.1", - ) - go_repository( - name = "com_github_hashicorp_logutils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/logutils", - sum = "h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_hashicorp_terraform_exec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-exec", - sum = "h1:FpqZ6n50Tk95mItTSS9BjeOVUb4eg81SpgVtZNNtFSM=", - version = "v0.19.0", - ) - go_repository( - name = "com_github_hashicorp_terraform_json", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-json", - sum = "h1:pCjgJEqqDESv4y0Tzdqfxr/edOIGkjs8keY42xfNBwU=", - version = "v0.18.0", - ) - go_repository( - name = "com_github_hashicorp_terraform_plugin_framework", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-plugin-framework", - sum = "h1:P7a7VP1GZbjc4rv921Xy5OckzhoiO3ig6SGxwelD2sI=", - version = "v1.4.2", - ) - go_repository( - name = "com_github_hashicorp_terraform_plugin_framework_validators", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-plugin-framework-validators", - sum = "h1:HOjBuMbOEzl7snOdOoUfE2Jgeto6JOjLVQ39Ls2nksc=", - version = "v0.12.0", - ) - go_repository( - name = "com_github_hashicorp_terraform_plugin_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-plugin-go", - sum = "h1:lf/jTGTeELcz5IIbn/94mJdmnTjRYm6S6ct/JqCSr50=", - version = "v0.19.1", - ) - go_repository( - name = "com_github_hashicorp_terraform_plugin_log", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-plugin-log", - sum = "h1:i7hOA+vdAItN1/7UrfBqBwvYPQ9TFvymaRGZED3FCV0=", - version = "v0.9.0", - ) - go_repository( - name = "com_github_hashicorp_terraform_plugin_sdk_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-plugin-sdk/v2", - sum = "h1:wcOKYwPI9IorAJEBLzgclh3xVolO7ZorYd6U1vnok14=", - version = "v2.29.0", - ) - go_repository( - name = "com_github_hashicorp_terraform_plugin_testing", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-plugin-testing", - sum = "h1:T4aQh9JAhmWo4+t1A7x+rnxAJHCDIYW9kXyo4sVO92c=", - version = "v1.5.1", - ) - go_repository( - name = "com_github_hashicorp_terraform_registry_address", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-registry-address", - sum = "h1:2TAiKJ1A3MAkZlH1YI/aTVcLZRu7JseiXNRHbOAyoTI=", - version = "v0.2.3", - ) - go_repository( - name = "com_github_hashicorp_terraform_svchost", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/terraform-svchost", - sum = "h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=", - version = "v0.1.1", - ) - go_repository( - name = "com_github_hashicorp_vault_api", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/vault/api", - sum = "h1:YjkZLJ7K3inKgMZ0wzCU9OHqc+UqMQyXsPXnf3Cl2as=", - version = "v1.9.2", - ) - go_repository( - name = "com_github_hashicorp_yamux", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/yamux", - sum = "h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=", - version = "v0.0.0-20181012175058-2f1d1f20f75d", - ) - go_repository( - name = "com_github_hexops_gotextdiff", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hexops/gotextdiff", - sum = "h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=", - version = "v1.0.3", - ) - go_repository( - name = "com_github_honeycombio_beeline_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/honeycombio/beeline-go", - sum = "h1:cUDe555oqvw8oD76BQJ8alk7FP0JZ/M/zXpNvOEDLDc=", - version = "v1.10.0", - ) - go_repository( - name = "com_github_honeycombio_libhoney_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/honeycombio/libhoney-go", - sum = "h1:kPpqoz6vbOzgp7jC6SR7SkNj7rua7rgxvznI6M3KdHc=", - version = "v1.16.0", - ) - go_repository( - name = "com_github_howeyc_gopass", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/howeyc/gopass", - sum = "h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM=", - version = "v0.0.0-20210920133722-c8aef6fb66ef", - ) - go_repository( - name = "com_github_hpcloud_tail", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/hpcloud/tail", - sum = "h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_huandu_xstrings", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/huandu/xstrings", - sum = "h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=", - version = "v1.4.0", - ) - go_repository( - name = "com_github_iancoleman_strcase", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/iancoleman/strcase", - sum = "h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=", - version = "v0.2.0", - ) - go_repository( - name = "com_github_ianlancetaylor_demangle", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ianlancetaylor/demangle", - sum = "h1:rcanfLhLDA8nozr/K289V1zcntHr3V+SHlXwzz1ZI2g=", - version = "v0.0.0-20220319035150-800ac71e25c2", - ) - go_repository( - name = "com_github_imdario_mergo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/imdario/mergo", - sum = "h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=", - version = "v0.3.15", - ) - go_repository( - name = "com_github_in_toto_in_toto_golang", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/in-toto/in-toto-golang", - sum = "h1:tHny7ac4KgtsfrG6ybU8gVOZux2H8jN05AXJ9EBM1XU=", - version = "v0.9.0", - ) - go_repository( - name = "com_github_inconshreveable_mousetrap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/inconshreveable/mousetrap", - sum = "h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_intel_goresctrl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/intel/goresctrl", - sum = "h1:K2D3GOzihV7xSBedGxONSlaw/un1LZgWsc9IfqipN4c=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_ishidawataru_sctp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ishidawataru/sctp", - sum = "h1:qPmlgoeRS18y2dT+iAH5vEKZgIqgiPi2Y8UCu/b7Aq8=", - version = "v0.0.0-20190723014705-7c296d48a2b5", - ) - go_repository( - name = "com_github_jbenet_go_context", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jbenet/go-context", - sum = "h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=", - version = "v0.0.0-20150711004518-d14ea06fba99", - ) - go_repository( - name = "com_github_jedisct1_go_minisign", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jedisct1/go-minisign", - sum = "h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8=", - version = "v0.0.0-20211028175153-1c139d1cc84b", - ) - go_repository( - name = "com_github_jeffashton_win_pdh", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/JeffAshton/win_pdh", - sum = "h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI=", - version = "v0.0.0-20161109143554-76bb4ee9f0ab", - ) - go_repository( - name = "com_github_jellydator_ttlcache_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jellydator/ttlcache/v3", - sum = "h1:cHgCSMS7TdQcoprXnWUptJZzyFsqs18Lt8VVhRuZYVU=", - version = "v3.0.1", - ) - go_repository( - name = "com_github_jessevdk_go_flags", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jessevdk/go-flags", - sum = "h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=", - version = "v1.4.0", - ) - go_repository( - name = "com_github_jhump_protoreflect", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jhump/protoreflect", - sum = "h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c=", - version = "v1.15.1", - ) - go_repository( - name = "com_github_jmespath_go_jmespath", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jmespath/go-jmespath", - sum = "h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_jmespath_go_jmespath_internal_testify", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jmespath/go-jmespath/internal/testify", - sum = "h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=", - version = "v1.5.1", - ) - go_repository( - name = "com_github_jmhodges_clock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jmhodges/clock", - sum = "h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4=", - version = "v0.0.0-20160418191101-880ee4c33548", - ) - go_repository( - name = "com_github_jmoiron_sqlx", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jmoiron/sqlx", - sum = "h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=", - version = "v1.3.5", - ) - go_repository( - name = "com_github_joho_godotenv", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/joho/godotenv", - sum = "h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_jonboulle_clockwork", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jonboulle/clockwork", - sum = "h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_josharian_intern", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/josharian/intern", - sum = "h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_josharian_native", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/josharian/native", - sum = "h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_jpillora_backoff", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jpillora/backoff", - sum = "h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_jsimonetti_rtnetlink", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jsimonetti/rtnetlink", - sum = "h1:Bl3VxrWwi3eNj2pFuG2x3xcIArSAvHf9paz1OXiDT9A=", - version = "v1.3.1", - ) - go_repository( - name = "com_github_json_iterator_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/json-iterator/go", - sum = "h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=", - version = "v1.1.12", - ) - go_repository( - name = "com_github_jstemmer_go_junit_report", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/jstemmer/go-junit-report", - sum = "h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=", - version = "v0.9.1", - ) - go_repository( - name = "com_github_julienschmidt_httprouter", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/julienschmidt/httprouter", - sum = "h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_k0kubun_go_ansi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/k0kubun/go-ansi", - sum = "h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=", - version = "v0.0.0-20180517002512-3bf9e2903213", - ) - go_repository( - name = "com_github_karrick_godirwalk", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/karrick/godirwalk", - sum = "h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=", - version = "v1.17.0", - ) - go_repository( - name = "com_github_katexochen_sh_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/katexochen/sh/v3", - sum = "h1:jrU9BWBgp9o2NcetUVm3dNpQ2SK1zG6aF6WF0wtPajc=", - version = "v3.7.0", - ) - go_repository( - name = "com_github_kevinburke_ssh_config", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kevinburke/ssh_config", - sum = "h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_kisielk_errcheck", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kisielk/errcheck", - sum = "h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY=", - version = "v1.5.0", - ) - go_repository( - name = "com_github_kisielk_gotool", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kisielk/gotool", - sum = "h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_klauspost_asmfmt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/klauspost/asmfmt", - sum = "h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=", - version = "v1.3.2", - ) - go_repository( - name = "com_github_klauspost_compress", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/klauspost/compress", - sum = "h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=", - version = "v1.16.5", - ) - go_repository( - name = "com_github_klauspost_cpuid_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/klauspost/cpuid/v2", - sum = "h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=", - version = "v2.0.9", - ) - go_repository( - name = "com_github_konsorten_go_windows_terminal_sequences", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/konsorten/go-windows-terminal-sequences", - sum = "h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_kr_fs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kr/fs", - sum = "h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_kr_logfmt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kr/logfmt", - sum = "h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=", - version = "v0.0.0-20140226030751-b84e30acd515", - ) - go_repository( - name = "com_github_kr_pretty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kr/pretty", - sum = "h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=", - version = "v0.3.1", - ) - go_repository( - name = "com_github_kr_pty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kr/pty", - sum = "h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=", - version = "v1.1.8", - ) - go_repository( - name = "com_github_kr_text", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kr/text", - sum = "h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=", - version = "v0.2.0", - ) - go_repository( - name = "com_github_kylelemons_godebug", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/kylelemons/godebug", - sum = "h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_lann_builder", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lann/builder", - sum = "h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=", - version = "v0.0.0-20180802200727-47ae307949d0", - ) - go_repository( - name = "com_github_lann_ps", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lann/ps", - sum = "h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=", - version = "v0.0.0-20150810152359-62de8c46ede0", - ) - go_repository( - name = "com_github_leodido_go_urn", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/leodido/go-urn", - sum = "h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=", - version = "v1.2.4", - ) - go_repository( - name = "com_github_lestrrat_go_backoff_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lestrrat-go/backoff/v2", - sum = "h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=", - version = "v2.0.8", - ) - go_repository( - name = "com_github_lestrrat_go_blackmagic", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lestrrat-go/blackmagic", - sum = "h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_lestrrat_go_httpcc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lestrrat-go/httpcc", - sum = "h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_lestrrat_go_iter", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lestrrat-go/iter", - sum = "h1:q8faalr2dY6o8bV45uwrxq12bRa1ezKrB6oM9FUgN4A=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_lestrrat_go_jwx", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lestrrat-go/jwx", - sum = "h1:tAx93jN2SdPvFn08fHNAhqFJazn5mBBOB8Zli0g0otA=", - version = "v1.2.25", - ) - go_repository( - name = "com_github_lestrrat_go_option", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lestrrat-go/option", - sum = "h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_letsencrypt_boulder", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/letsencrypt/boulder", - sum = "h1:ndns1qx/5dL43g16EQkPV/i8+b3l5bYQwLeoSBe7tS8=", - version = "v0.0.0-20221109233200-85aa52084eaf", - ) - go_repository( - name = "com_github_letsencrypt_challtestsrv", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/letsencrypt/challtestsrv", - sum = "h1:Lzv4jM+wSgVMCeO5a/F/IzSanhClstFMnX6SfrAJXjI=", - version = "v1.2.1", - ) - go_repository( - name = "com_github_letsencrypt_pkcs11key_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/letsencrypt/pkcs11key/v4", - sum = "h1:qLc/OznH7xMr5ARJgkZCCWk+EomQkiNTOoOF5LAgagc=", - version = "v4.0.0", - ) - go_repository( - name = "com_github_lib_pq", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lib/pq", - sum = "h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=", - version = "v1.10.9", - ) - go_repository( - name = "com_github_libopenstorage_openstorage", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/libopenstorage/openstorage", - sum = "h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_liggitt_tabwriter", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/liggitt/tabwriter", - sum = "h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=", - version = "v0.0.0-20181228230101-89fcab3d43de", - ) - go_repository( - name = "com_github_linuxkit_virtsock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/linuxkit/virtsock", - sum = "h1:jUp75lepDg0phMUJBCmvaeFDldD2N3S1lBuPwUTszio=", - version = "v0.0.0-20201010232012-f8cee7dfc7a3", - ) - go_repository( - name = "com_github_lithammer_dedent", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lithammer/dedent", - sum = "h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_lyft_protoc_gen_star_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/lyft/protoc-gen-star/v2", - sum = "h1:/3+/2sWyXeMLzKd1bX+ixWKgEMsULrIivpDsuaF441o=", - version = "v2.0.3", - ) - go_repository( - name = "com_github_magiconair_properties", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/magiconair/properties", - sum = "h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=", - version = "v1.8.7", - ) - go_repository( - name = "com_github_mailru_easyjson", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mailru/easyjson", - sum = "h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=", - version = "v0.7.7", - ) - go_repository( - name = "com_github_makenowjust_heredoc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/MakeNowJust/heredoc", - sum = "h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_markbates_errx", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/markbates/errx", - sum = "h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_markbates_oncer", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/markbates/oncer", - sum = "h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_markbates_safe", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/markbates/safe", - sum = "h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_martinjungblut_go_cryptsetup", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/martinjungblut/go-cryptsetup", - patches = [ - "//3rdparty/bazel/com_github_martinjungblut_go_cryptsetup:com_github_martinjungblut_go_cryptsetup.patch", # keep - ], - replace = "github.com/daniel-weisse/go-cryptsetup", - sum = "h1:ToajP6trZoiqlZ3Z4uoG1P02/wtqSw1AcowOXOYjATk=", - version = "v0.0.0-20230705150314-d8c07bd1723c", - ) - go_repository( - name = "com_github_masterminds_goutils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Masterminds/goutils", - sum = "h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_masterminds_semver_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Masterminds/semver/v3", - sum = "h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=", - version = "v3.2.1", - ) - go_repository( - name = "com_github_masterminds_sprig_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Masterminds/sprig/v3", - sum = "h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=", - version = "v3.2.3", - ) - go_repository( - name = "com_github_masterminds_squirrel", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Masterminds/squirrel", - sum = "h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=", - version = "v1.5.4", - ) - go_repository( - name = "com_github_masterminds_vcs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Masterminds/vcs", - sum = "h1:IIA2aBdXvfbIM+yl/eTnL4hb1XwdpvuQLglAix1gweE=", - version = "v1.13.3", - ) - go_repository( - name = "com_github_mattn_go_colorable", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-colorable", - sum = "h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=", - version = "v0.1.13", - ) - go_repository( - name = "com_github_mattn_go_isatty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-isatty", - sum = "h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=", - version = "v0.0.19", - ) - go_repository( - name = "com_github_mattn_go_oci8", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-oci8", - sum = "h1:aEUDxNAyDG0tv8CA3TArnDQNyc4EhnWlsfxRgDHABHM=", - version = "v0.1.1", - ) - go_repository( - name = "com_github_mattn_go_runewidth", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-runewidth", - sum = "h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=", - version = "v0.0.14", - ) - go_repository( - name = "com_github_mattn_go_shellwords", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-shellwords", - sum = "h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=", - version = "v1.0.12", - ) - go_repository( - name = "com_github_mattn_go_sqlite3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-sqlite3", - sum = "h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=", - version = "v1.14.15", - ) - go_repository( - name = "com_github_matttproud_golang_protobuf_extensions", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/matttproud/golang_protobuf_extensions", - sum = "h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=", - version = "v1.0.4", - ) - go_repository( - name = "com_github_mdlayher_ethtool", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mdlayher/ethtool", - sum = "h1:Y7LoKqIgD7vmqJ7+6ZVnADuwUO+m3tGXbf2lK0OvjIw=", - version = "v0.0.0-20221212131811-ba3b4bc2e02c", - ) - go_repository( - name = "com_github_mdlayher_genetlink", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mdlayher/genetlink", - sum = "h1:roBiPnual+eqtRkKX2Jb8UQN5ZPWnhDCGj/wR6Jlz2w=", - version = "v1.3.1", - ) - go_repository( - name = "com_github_mdlayher_netlink", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mdlayher/netlink", - sum = "h1:FdUaT/e33HjEXagwELR8R3/KL1Fq5x3G5jgHLp/BTmg=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_mdlayher_socket", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mdlayher/socket", - sum = "h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_microsoft_applicationinsights_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/microsoft/ApplicationInsights-Go", - sum = "h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY=", - version = "v0.4.4", - ) - go_repository( - name = "com_github_microsoft_go_winio", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Microsoft/go-winio", - sum = "h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=", - version = "v0.6.1", - ) - go_repository( - name = "com_github_microsoft_hcsshim", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Microsoft/hcsshim", - sum = "h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM=", - version = "v0.11.0", - ) - go_repository( - name = "com_github_miekg_dns", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/miekg/dns", - sum = "h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=", - version = "v1.1.50", - ) - go_repository( - name = "com_github_miekg_pkcs11", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/miekg/pkcs11", - sum = "h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_minio_asm2plan9s", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/minio/asm2plan9s", - sum = "h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=", - version = "v0.0.0-20200509001527-cdd76441f9d8", - ) - go_repository( - name = "com_github_minio_c2goasm", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/minio/c2goasm", - sum = "h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=", - version = "v0.0.0-20190812172519-36a3d3bbc4f3", - ) - go_repository( - name = "com_github_minio_sha256_simd", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/minio/sha256-simd", - sum = "h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_mistifyio_go_zfs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mistifyio/go-zfs", - sum = "h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk=", - version = "v2.1.2-0.20190413222219-f784269be439+incompatible", - ) - go_repository( - name = "com_github_mistifyio_go_zfs_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mistifyio/go-zfs/v3", - sum = "h1:YaoXgBePoMA12+S1u/ddkv+QqxcfiZK4prI6HPnkFiU=", - version = "v3.0.1", - ) - go_repository( - name = "com_github_mitchellh_cli", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/cli", - sum = "h1:OxRIeJXpAMztws/XHlN2vu6imG5Dpq+j61AzAX5fLng=", - version = "v1.1.5", - ) - go_repository( - name = "com_github_mitchellh_colorstring", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/colorstring", - sum = "h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=", - version = "v0.0.0-20190213212951-d06e56a500db", - ) - go_repository( - name = "com_github_mitchellh_copystructure", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/copystructure", - sum = "h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_mitchellh_go_homedir", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/go-homedir", - sum = "h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_mitchellh_go_testing_interface", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/go-testing-interface", - sum = "h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=", - version = "v1.14.1", - ) - go_repository( - name = "com_github_mitchellh_go_wordwrap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/go-wordwrap", - sum = "h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_mitchellh_mapstructure", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/mapstructure", - sum = "h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=", - version = "v1.5.0", - ) - go_repository( - name = "com_github_mitchellh_reflectwalk", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mitchellh/reflectwalk", - sum = "h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_moby_ipvs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/ipvs", - sum = "h1:ONN4pGaZQgAx+1Scz5RvWV4Q7Gb+mvfRh3NsPS+1XQQ=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_moby_locker", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/locker", - sum = "h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=", - version = "v1.0.1", - ) - go_repository( - name = "com_github_moby_spdystream", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/spdystream", - sum = "h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=", - version = "v0.2.0", - ) - go_repository( - name = "com_github_moby_sys_mountinfo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/sys/mountinfo", - sum = "h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=", - version = "v0.6.2", - ) - go_repository( - name = "com_github_moby_sys_sequential", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/sys/sequential", - sum = "h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_moby_sys_signal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/sys/signal", - sum = "h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI=", - version = "v0.7.0", - ) - go_repository( - name = "com_github_moby_sys_symlink", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/sys/symlink", - sum = "h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc=", - version = "v0.2.0", - ) - go_repository( - name = "com_github_moby_term", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/moby/term", - sum = "h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_modern_go_concurrent", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/modern-go/concurrent", - sum = "h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=", - version = "v0.0.0-20180306012644-bacd9c7ef1dd", - ) - go_repository( - name = "com_github_modern_go_reflect2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/modern-go/reflect2", - sum = "h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_mohae_deepcopy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mohae/deepcopy", - sum = "h1:e+l77LJOEqXTIQihQJVkA6ZxPOUmfPM5e4H7rcpgtSk=", - version = "v0.0.0-20170603005431-491d3605edfb", - ) - go_repository( - name = "com_github_monochromegane_go_gitignore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/monochromegane/go-gitignore", - sum = "h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=", - version = "v0.0.0-20200626010858-205db1a8cc00", - ) - go_repository( - name = "com_github_montanaflynn_stats", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/montanaflynn/stats", - sum = "h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU=", - version = "v0.7.0", - ) - go_repository( - name = "com_github_morikuni_aec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/morikuni/aec", - sum = "h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_mr_tron_base58", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mr-tron/base58", - sum = "h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_mrunalp_fileutils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mrunalp/fileutils", - sum = "h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_munnerz_goautoneg", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/munnerz/goautoneg", - sum = "h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=", - version = "v0.0.0-20191010083416-a7dc8b61c822", - ) - go_repository( - name = "com_github_mwitkow_go_conntrack", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mwitkow/go-conntrack", - sum = "h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=", - version = "v0.0.0-20190716064945-2f068394615f", - ) - go_repository( - name = "com_github_mxk_go_flowrate", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/mxk/go-flowrate", - sum = "h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=", - version = "v0.0.0-20140419014527-cca7078d478f", - ) - go_repository( - name = "com_github_nelsam_hel_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/nelsam/hel/v2", - sum = "h1:Z3TAKd9JS3BoKi6fW+d1bKD2Mf0FzTqDUEAwLWzYPRQ=", - version = "v2.3.3", - ) - go_repository( - name = "com_github_niemeyer_pretty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/niemeyer/pretty", - sum = "h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=", - version = "v0.0.0-20200227124842-a10e7caefd8e", - ) - go_repository( - name = "com_github_nytimes_gziphandler", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/NYTimes/gziphandler", - sum = "h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_oklog_run", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/oklog/run", - sum = "h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_oklog_ulid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/oklog/ulid", - sum = "h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=", - version = "v1.3.1", - ) - go_repository( - name = "com_github_olekukonko_tablewriter", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/olekukonko/tablewriter", - sum = "h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=", - version = "v0.0.5", - ) - go_repository( - name = "com_github_oneofone_xxhash", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/OneOfOne/xxhash", - sum = "h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=", - version = "v1.2.8", - ) - go_repository( - name = "com_github_onsi_ginkgo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/onsi/ginkgo", - sum = "h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=", - version = "v1.8.0", - ) - go_repository( - name = "com_github_onsi_ginkgo_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/onsi/ginkgo/v2", - sum = "h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=", - version = "v2.13.0", - ) - go_repository( - name = "com_github_onsi_gomega", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/onsi/gomega", - sum = "h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=", - version = "v1.27.10", - ) - go_repository( - name = "com_github_open_policy_agent_opa", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/open-policy-agent/opa", - sum = "h1:qocVAKyjrqMjCqsU02S/gHyLr4AQQ9xMtuV1kKnnyhM=", - version = "v0.42.2", - ) - go_repository( - name = "com_github_opencontainers_go_digest", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opencontainers/go-digest", - sum = "h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_opencontainers_image_spec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opencontainers/image-spec", - sum = "h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=", - version = "v1.1.0-rc5", - ) - go_repository( - name = "com_github_opencontainers_runc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opencontainers/runc", - sum = "h1:XbhB8IfG/EsnhNvZtNdLB0GBw92GYEFvKlhaJk9jUgA=", - version = "v1.1.6", - ) - go_repository( - name = "com_github_opencontainers_runtime_spec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opencontainers/runtime-spec", - sum = "h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w=", - version = "v1.1.0-rc.1", - ) - go_repository( - name = "com_github_opencontainers_runtime_tools", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opencontainers/runtime-tools", - sum = "h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=", - version = "v0.9.1-0.20221107090550-2e043c6bd626", - ) - go_repository( - name = "com_github_opencontainers_selinux", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opencontainers/selinux", - sum = "h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=", - version = "v1.11.0", - ) - go_repository( - name = "com_github_opentracing_opentracing_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/opentracing/opentracing-go", - sum = "h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_otiai10_copy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/otiai10/copy", - sum = "h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ=", - version = "v1.6.0", - ) - go_repository( - name = "com_github_otiai10_curr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/otiai10/curr", - sum = "h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_otiai10_mint", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/otiai10/mint", - sum = "h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=", - version = "v1.3.2", - ) - go_repository( - name = "com_github_pborman_uuid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pborman/uuid", - sum = "h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=", - version = "v1.2.1", - ) - go_repository( - name = "com_github_pelletier_go_buffruneio", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pelletier/go-buffruneio", - sum = "h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=", - version = "v0.2.0", - ) - go_repository( - name = "com_github_pelletier_go_toml", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pelletier/go-toml", - sum = "h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=", - version = "v1.9.5", - ) - go_repository( - name = "com_github_pelletier_go_toml_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pelletier/go-toml/v2", - sum = "h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=", - version = "v2.0.8", - ) - go_repository( - name = "com_github_peterbourgon_diskv", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/peterbourgon/diskv", - sum = "h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=", - version = "v2.0.1+incompatible", - ) - go_repository( - name = "com_github_phayes_freeport", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/phayes/freeport", - sum = "h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=", - version = "v0.0.0-20220201140144-74d24b5ae9f5", - ) - go_repository( - name = "com_github_pierrec_lz4_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pierrec/lz4/v4", - sum = "h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=", - version = "v4.1.15", - ) - go_repository( - name = "com_github_pjbgf_sha1cd", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pjbgf/sha1cd", - sum = "h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=", - version = "v0.3.0", - ) - go_repository( - name = "com_github_pkg_browser", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pkg/browser", - sum = "h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=", - version = "v0.0.0-20210911075715-681adbf594b8", - ) - go_repository( - name = "com_github_pkg_diff", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pkg/diff", - sum = "h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=", - version = "v0.0.0-20210226163009-20ebb0f2a09e", - ) - go_repository( - name = "com_github_pkg_errors", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pkg/errors", - sum = "h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=", - version = "v0.9.1", - ) - go_repository( - name = "com_github_pkg_sftp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pkg/sftp", - sum = "h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs=", - version = "v1.13.1", - ) - go_repository( - name = "com_github_pmezard_go_difflib", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pmezard/go-difflib", - sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_posener_complete", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/posener/complete", - sum = "h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=", - version = "v1.2.3", - ) - go_repository( - name = "com_github_poy_onpar", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/poy/onpar", - sum = "h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=", - version = "v1.1.2", - ) - go_repository( - name = "com_github_pquerna_cachecontrol", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/pquerna/cachecontrol", - sum = "h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_prometheus_client_golang", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/prometheus/client_golang", - sum = "h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=", - version = "v1.16.0", - ) - go_repository( - name = "com_github_prometheus_client_model", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/prometheus/client_model", - sum = "h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_prometheus_common", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/prometheus/common", - sum = "h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=", - version = "v0.44.0", - ) - go_repository( - name = "com_github_prometheus_procfs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/prometheus/procfs", - sum = "h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=", - version = "v0.10.1", - ) - go_repository( - name = "com_github_prometheus_prometheus", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/prometheus/prometheus", - sum = "h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg=", - version = "v2.5.0+incompatible", - ) - go_repository( - name = "com_github_protonmail_go_crypto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ProtonMail/go-crypto", - sum = "h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg=", - version = "v0.0.0-20230828082145-3c4c8a2d2371", - ) - go_repository( - name = "com_github_protonmail_go_mime", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ProtonMail/go-mime", - sum = "h1:dS7r5z4iGS0qCjM7UwWdsEMzQesUQbGcXdSm2/tWboA=", - version = "v0.0.0-20221031134845-8fd9bc37cf08", - ) - go_repository( - name = "com_github_protonmail_gopenpgp_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ProtonMail/gopenpgp/v2", - sum = "h1:97SjlWNAxXl9P22lgwgrZRshQdiEfAht0g3ZoiA1GCw=", - version = "v2.5.2", - ) - go_repository( - name = "com_github_puerkitobio_purell", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/PuerkitoBio/purell", - sum = "h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_puerkitobio_urlesc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/PuerkitoBio/urlesc", - sum = "h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=", - version = "v0.0.0-20170810143723-de5bf2ad4578", - ) - go_repository( - name = "com_github_rcrowley_go_metrics", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rcrowley/go-metrics", - sum = "h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=", - version = "v0.0.0-20200313005456-10cdbea86bc0", - ) - go_repository( - name = "com_github_redis_go_redis_v9", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/redis/go-redis/v9", - sum = "h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=", - version = "v9.0.5", - ) - go_repository( - name = "com_github_rivo_uniseg", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rivo/uniseg", - sum = "h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=", - version = "v0.4.4", - ) - go_repository( - name = "com_github_robfig_cron_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/robfig/cron/v3", - sum = "h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=", - version = "v3.0.1", - ) - go_repository( - name = "com_github_rogpeppe_fastuuid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rogpeppe/fastuuid", - sum = "h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_rogpeppe_go_internal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rogpeppe/go-internal", - sum = "h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=", - version = "v1.11.0", - ) - go_repository( - name = "com_github_rs_cors", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rs/cors", - sum = "h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=", - version = "v1.9.0", - ) - go_repository( - name = "com_github_rubenv_sql_migrate", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rubenv/sql-migrate", - sum = "h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=", - version = "v1.5.2", - ) - go_repository( - name = "com_github_rubiojr_go_vhd", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/rubiojr/go-vhd", - sum = "h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc=", - version = "v0.0.0-20200706105327-02e210299021", - ) - go_repository( - name = "com_github_russross_blackfriday", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/russross/blackfriday", - sum = "h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=", - version = "v1.6.0", - ) - go_repository( - name = "com_github_russross_blackfriday_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/russross/blackfriday/v2", - sum = "h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=", - version = "v2.1.0", - ) - go_repository( - name = "com_github_ryanuber_go_glob", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ryanuber/go-glob", - sum = "h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_sassoftware_relic", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sassoftware/relic", - sum = "h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A=", - version = "v7.2.1+incompatible", - ) - go_repository( - name = "com_github_sassoftware_relic_v7", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sassoftware/relic/v7", - sum = "h1:2ZUM6ovo3STCAp0hZnO9nQY9lOB8OyfneeYIi4YUxMU=", - version = "v7.5.5", - ) - go_repository( - name = "com_github_schollz_progressbar_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/schollz/progressbar/v3", - sum = "h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=", - version = "v3.13.1", - ) - go_repository( - name = "com_github_sebdah_goldie", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sebdah/goldie", - sum = "h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_seccomp_libseccomp_golang", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/seccomp/libseccomp-golang", - sum = "h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds=", - version = "v0.9.2-0.20220502022130-f33da4d89646", - ) - go_repository( - name = "com_github_secure_systems_lab_go_securesystemslib", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/secure-systems-lab/go-securesystemslib", - sum = "h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=", - version = "v0.6.0", - ) - go_repository( - name = "com_github_segmentio_ksuid", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/segmentio/ksuid", - sum = "h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=", - version = "v1.0.4", - ) - go_repository( - name = "com_github_sergi_go_diff", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sergi/go-diff", - sum = "h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=", - version = "v1.3.1", - ) - go_repository( - name = "com_github_shibumi_go_pathspec", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/shibumi/go-pathspec", - sum = "h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_shopify_logrus_bugsnag", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/Shopify/logrus-bugsnag", - sum = "h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=", - version = "v0.0.0-20171204204709-577dee27f20d", - ) - go_repository( - name = "com_github_shopspring_decimal", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/shopspring/decimal", - sum = "h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=", - version = "v1.3.1", - ) - go_repository( - name = "com_github_siderolabs_crypto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/crypto", - sum = "h1:o1KIR1KyevUcY9nbJlSyQAj7+p+rveGGF8LjAAFMtjc=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_siderolabs_gen", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/gen", - sum = "h1:V3UsZ2KrsryaTMZGZUHAr1CFdPc2/R1lM6lA4a4zCDo=", - version = "v0.4.3", - ) - go_repository( - name = "com_github_siderolabs_go_api_signature", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/go-api-signature", - sum = "h1:C5tUzuFsJYidpYyVfJGYpgQwETglA8B62ET4obkLDGE=", - version = "v0.2.2", - ) - go_repository( - name = "com_github_siderolabs_go_blockdevice", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/go-blockdevice", - sum = "h1:NgpR9XTl/N7WeL59QHBsseDD0Nb8Y2nel+W3u7xHIvY=", - version = "v0.4.5", - ) - go_repository( - name = "com_github_siderolabs_go_debug", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/go-debug", - sum = "h1:c8styCvp+MO0oPO8q4N1CKSF3fVuAT0qnuUIeZ/BiW0=", - version = "v0.2.2", - ) - go_repository( - name = "com_github_siderolabs_go_pointer", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/go-pointer", - sum = "h1:6TshPKep2doDQJAAtHUuHWXbca8ZfyRySjSBT/4GsMU=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_siderolabs_net", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/net", - sum = "h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_siderolabs_protoenc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/protoenc", - sum = "h1:QFxWIAo//12+/bm27GNYoK/TpQGTYsRrrZCu9jSghvU=", - version = "v0.2.0", - ) - go_repository( - name = "com_github_siderolabs_talos_pkg_machinery", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/siderolabs/talos/pkg/machinery", - sum = "h1:SX7Q6FxTDyX2hxugMgIqyivXWzemgMhHj3AlDbxjuFw=", - version = "v1.4.6", - ) - go_repository( - name = "com_github_sigstore_protobuf_specs", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/protobuf-specs", - sum = "h1:X0l/E2C2c79t/rI/lmSu8WAoKWsQtMqDzAMiDdEMGr8=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_sigstore_rekor", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/rekor", - sum = "h1:5JK/zKZvcQpL/jBmHvmFj3YbpDMBQnJQ6ygp8xdF3bY=", - version = "v1.2.2", - ) - go_repository( - name = "com_github_sigstore_sigstore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/sigstore", - sum = "h1:fCATemikcBK0cG4+NcM940MfoIgmioY1vC6E66hXxks=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_sigstore_sigstore_pkg_signature_kms_aws", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/sigstore/pkg/signature/kms/aws", - sum = "h1:rDHrG/63b3nBq3G9plg7iYnWN6lBhOfq/XultlCZgII=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_sigstore_sigstore_pkg_signature_kms_azure", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/sigstore/pkg/signature/kms/azure", - sum = "h1:X3ezwolP+b1jP3R6XPOWhUU0TZKONiv6EIRuySlZGrY=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_sigstore_sigstore_pkg_signature_kms_gcp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/sigstore/pkg/signature/kms/gcp", - sum = "h1:mj1KhdzzP1me994bt1UXhq5KZGSR1SoqxTqcT+hfPMk=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_sigstore_sigstore_pkg_signature_kms_hashivault", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sigstore/sigstore/pkg/signature/kms/hashivault", - sum = "h1:fhOToGY5fC5TY101an8i/oDYpoLzUJ1nUFwhnHA1+XY=", - version = "v1.7.1", - ) - go_repository( - name = "com_github_sirupsen_logrus", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/sirupsen/logrus", - sum = "h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=", - version = "v1.9.3", - ) - go_repository( - name = "com_github_skeema_knownhosts", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/skeema/knownhosts", - sum = "h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_skratchdot_open_golang", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/skratchdot/open-golang", - sum = "h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=", - version = "v0.0.0-20200116055534-eef842397966", - ) - go_repository( - name = "com_github_soheilhy_cmux", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/soheilhy/cmux", - sum = "h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=", - version = "v0.1.5", - ) - go_repository( - name = "com_github_spaolacci_murmur3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spaolacci/murmur3", - sum = "h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=", - version = "v0.0.0-20180118202830-f09979ecbc72", - ) - go_repository( - name = "com_github_spf13_afero", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spf13/afero", - sum = "h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY=", - version = "v1.10.0", - ) - go_repository( - name = "com_github_spf13_cast", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spf13/cast", - sum = "h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=", - version = "v1.5.1", - ) - go_repository( - name = "com_github_spf13_cobra", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spf13/cobra", - sum = "h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=", - version = "v1.7.0", - ) - go_repository( - name = "com_github_spf13_jwalterweatherman", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spf13/jwalterweatherman", - sum = "h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_spf13_pflag", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spf13/pflag", - sum = "h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=", - version = "v1.0.5", - ) - go_repository( - name = "com_github_spf13_viper", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/spf13/viper", - sum = "h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc=", - version = "v1.16.0", - ) - go_repository( - name = "com_github_src_d_gcfg", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/src-d/gcfg", - sum = "h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=", - version = "v1.4.0", - ) - go_repository( - name = "com_github_stefanberger_go_pkcs11uri", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/stefanberger/go-pkcs11uri", - sum = "h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I=", - version = "v0.0.0-20201008174630-78d3cae3a980", - ) - go_repository( - name = "com_github_stoewer_go_strcase", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/stoewer/go-strcase", - sum = "h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=", - version = "v1.3.0", - ) - go_repository( - name = "com_github_stretchr_objx", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/stretchr/objx", - sum = "h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=", - version = "v0.5.0", - ) - go_repository( - name = "com_github_stretchr_testify", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/stretchr/testify", - sum = "h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=", - version = "v1.8.4", - ) - go_repository( - name = "com_github_subosito_gotenv", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/subosito/gotenv", - sum = "h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=", - version = "v1.4.2", - ) - go_repository( - name = "com_github_syndtr_gocapability", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/syndtr/gocapability", - sum = "h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=", - version = "v0.0.0-20200815063812-42c35b437635", - ) - go_repository( - name = "com_github_syndtr_goleveldb", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/syndtr/goleveldb", - sum = "h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=", - version = "v1.0.1-0.20220721030215-126854af5e6d", - ) - go_repository( - name = "com_github_tchap_go_patricia_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/tchap/go-patricia/v2", - sum = "h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes=", - version = "v2.3.1", - ) - go_repository( - name = "com_github_tedsuo_ifrit", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/tedsuo/ifrit", - sum = "h1:LUUe4cdABGrIJAhl1P1ZpWY76AwukVszFdwkVFVLwIk=", - version = "v0.0.0-20180802180643-bea94bb476cc", - ) - go_repository( - name = "com_github_theupdateframework_go_tuf", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/theupdateframework/go-tuf", - sum = "h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA=", - version = "v0.5.2", - ) - go_repository( - name = "com_github_tidwall_pretty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/tidwall/pretty", - sum = "h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_tink_crypto_tink_go_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/tink-crypto/tink-go/v2", - replace = "github.com/derpsteb/tink-go/v2", - sum = "h1:FVii9oXvddz9sFir5TRYjQKrzJLbVD/hibT+SnRSDzg=", - version = "v2.0.0-20231002051717-a808e454eed6", - ) - go_repository( - name = "com_github_titanous_rocacheck", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/titanous/rocacheck", - sum = "h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0=", - version = "v0.0.0-20171023193734-afe73141d399", - ) - go_repository( - name = "com_github_tmc_grpc_websocket_proxy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/tmc/grpc-websocket-proxy", - sum = "h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=", - version = "v0.0.0-20220101234140-673ab2c3ae75", - ) - go_repository( - name = "com_github_tomasen_realip", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/tomasen/realip", - sum = "h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc=", - version = "v0.0.0-20180522021738-f0c99a92ddce", - ) - go_repository( - name = "com_github_transparency_dev_merkle", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/transparency-dev/merkle", - sum = "h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=", - version = "v0.0.2", - ) - go_repository( - name = "com_github_ulikunitz_xz", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ulikunitz/xz", - sum = "h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=", - version = "v0.5.11", - ) - go_repository( - name = "com_github_urfave_cli", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/urfave/cli", - sum = "h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8=", - version = "v1.22.12", - ) - go_repository( - name = "com_github_vbatts_tar_split", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vbatts/tar-split", - sum = "h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=", - version = "v0.11.3", - ) - go_repository( - name = "com_github_vektah_gqlparser_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vektah/gqlparser/v2", - sum = "h1:C02NsyEsL4TXJB7ndonqTfuQOL4XPIu0aAWugdmTgmc=", - version = "v2.4.5", - ) - go_repository( - name = "com_github_veraison_go_cose", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/veraison/go-cose", - sum = "h1:AalPS4VGiKavpAzIlBjrn7bhqXiXi4jbMYY/2+UC+4o=", - version = "v1.1.0", - ) - go_repository( - name = "com_github_vincent_petithory_dataurl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vincent-petithory/dataurl", - sum = "h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_vishvananda_netlink", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vishvananda/netlink", - sum = "h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=", - version = "v1.2.1-beta.2", - ) - go_repository( - name = "com_github_vishvananda_netns", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vishvananda/netns", - sum = "h1:Cn05BRLm+iRP/DZxyVSsfVyrzgjDbwHwkVt38qvXnNI=", - version = "v0.0.2", - ) - go_repository( - name = "com_github_vmihailenco_msgpack", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vmihailenco/msgpack", - sum = "h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=", - version = "v4.0.4+incompatible", - ) - go_repository( - name = "com_github_vmihailenco_msgpack_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vmihailenco/msgpack/v5", - sum = "h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=", - version = "v5.4.1", - ) - go_repository( - name = "com_github_vmihailenco_tagparser_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vmihailenco/tagparser/v2", - sum = "h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=", - version = "v2.0.0", - ) - go_repository( - name = "com_github_vmware_govmomi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vmware/govmomi", - sum = "h1:O3tjSwQBy0XwI5uK1/yVIfQ1LP9bAECEDUfifnyGs9U=", - version = "v0.30.6", - ) - go_repository( - name = "com_github_vtolstov_go_ioctl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/vtolstov/go-ioctl", - sum = "h1:X6ps8XHfpQjw8dUStzlMi2ybiKQ2Fmdw7UM+TinwvyM=", - version = "v0.0.0-20151206205506-6be9cced4810", - ) - go_repository( - name = "com_github_weppos_publicsuffix_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/weppos/publicsuffix-go", - sum = "h1:eR9jm8DVMdrDUuVji4eOxPK4r/dANDlDBdISSUUV96s=", - version = "v0.20.1-0.20221031080346-e4081aa8a6de", - ) - go_repository( - name = "com_github_x448_float16", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/x448/float16", - sum = "h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=", - version = "v0.8.4", - ) - go_repository( - name = "com_github_xanzy_ssh_agent", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xanzy/ssh-agent", - sum = "h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=", - version = "v0.3.3", - ) - go_repository( - name = "com_github_xdg_go_pbkdf2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xdg-go/pbkdf2", - sum = "h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=", - version = "v1.0.0", - ) - go_repository( - name = "com_github_xdg_go_scram", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xdg-go/scram", - sum = "h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=", - version = "v1.1.1", - ) - go_repository( - name = "com_github_xdg_go_stringprep", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xdg-go/stringprep", - sum = "h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=", - version = "v1.0.3", - ) - go_repository( - name = "com_github_xeipuuv_gojsonpointer", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xeipuuv/gojsonpointer", - sum = "h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=", - version = "v0.0.0-20190905194746-02993c407bfb", - ) - go_repository( - name = "com_github_xeipuuv_gojsonreference", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xeipuuv/gojsonreference", - sum = "h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=", - version = "v0.0.0-20180127040603-bd5ef7bd5415", - ) - go_repository( - name = "com_github_xeipuuv_gojsonschema", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xeipuuv/gojsonschema", - sum = "h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_xhit_go_str2duration_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xhit/go-str2duration/v2", - sum = "h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=", - version = "v2.1.0", - ) - go_repository( - name = "com_github_xiang90_probing", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xiang90/probing", - sum = "h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=", - version = "v0.0.0-20190116061207-43a291ad63a2", - ) - go_repository( - name = "com_github_xlab_treeprint", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/xlab/treeprint", - sum = "h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=", - version = "v1.2.0", - ) - go_repository( - name = "com_github_yashtewari_glob_intersection", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/yashtewari/glob-intersection", - sum = "h1:6gJvMYQlTDOL3dMsPF6J0+26vwX9MB8/1q3uAdhmTrg=", - version = "v0.1.0", - ) - go_repository( - name = "com_github_youmark_pkcs8", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/youmark/pkcs8", - sum = "h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=", - version = "v0.0.0-20181117223130-1be2e3e5546d", - ) - go_repository( - name = "com_github_ysmood_fetchup", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ysmood/fetchup", - sum = "h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=", - version = "v0.2.3", - ) - go_repository( - name = "com_github_ysmood_goob", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ysmood/goob", - sum = "h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=", - version = "v0.4.0", - ) - go_repository( - name = "com_github_ysmood_got", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ysmood/got", - sum = "h1:IrV2uWLs45VXNvZqhJ6g2nIhY+pgIG1CUoOcqfXFl1s=", - version = "v0.34.1", - ) - go_repository( - name = "com_github_ysmood_gson", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ysmood/gson", - sum = "h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=", - version = "v0.7.3", - ) - go_repository( - name = "com_github_ysmood_leakless", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/ysmood/leakless", - sum = "h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=", - version = "v0.8.0", - ) - go_repository( - name = "com_github_yuin_goldmark", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/yuin/goldmark", - sum = "h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=", - version = "v1.4.13", - ) - go_repository( - name = "com_github_yvasiyarov_go_metrics", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/yvasiyarov/go-metrics", - sum = "h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=", - version = "v0.0.0-20140926110328-57bccd1ccd43", - ) - go_repository( - name = "com_github_yvasiyarov_gorelic", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/yvasiyarov/gorelic", - sum = "h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE=", - version = "v0.0.0-20141212073537-a9bba5b9ab50", - ) - go_repository( - name = "com_github_yvasiyarov_newrelic_platform_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/yvasiyarov/newrelic_platform_go", - sum = "h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY=", - version = "v0.0.0-20140908184405-b21fdbd4370f", - ) - go_repository( - name = "com_github_zalando_go_keyring", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/zalando/go-keyring", - sum = "h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4=", - version = "v0.2.2", - ) - go_repository( - name = "com_github_zclconf_go_cty", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/zclconf/go-cty", - sum = "h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA=", - version = "v1.14.1", - ) - go_repository( - name = "com_github_zclconf_go_cty_debug", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/zclconf/go-cty-debug", - sum = "h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI=", - version = "v0.0.0-20191215020915-b22d67c1ba0b", - ) - go_repository( - name = "com_github_zeebo_xxh3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/zeebo/xxh3", - sum = "h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=", - version = "v1.0.2", - ) - go_repository( - name = "com_github_zmap_zcrypto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/zmap/zcrypto", - sum = "h1:+nr36qrZEH0RIYNjcUEnOrCUdcSG3om2ANaFA6iSVWA=", - version = "v0.0.0-20220402174210-599ec18ecbac", - ) - go_repository( - name = "com_github_zmap_zlint_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "github.com/zmap/zlint/v3", - sum = "h1:Xs/lrMJY74MpJx/jSx2oVvZBrqlyUyFaLLBRyf68cqg=", - version = "v3.4.0", - ) - go_repository( - name = "com_google_cloud_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go", - sum = "h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME=", - version = "v0.110.8", - ) - go_repository( - name = "com_google_cloud_go_accessapproval", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/accessapproval", - sum = "h1:/5YjNhR6lzCvmJZAnByYkfEgWjfAKwYP6nkuTk6nKFE=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_accesscontextmanager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/accesscontextmanager", - sum = "h1:WIAt9lW9AXtqw/bnvrEUaE8VG/7bAAeMzRCBGMkc4+w=", - version = "v1.8.1", - ) - go_repository( - name = "com_google_cloud_go_aiplatform", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/aiplatform", - sum = "h1:J89aj+lqwtjn0qpQBMVaiOmDxBkKDEKUwl+GL19RRpc=", - version = "v1.50.0", - ) - go_repository( - name = "com_google_cloud_go_analytics", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/analytics", - sum = "h1:TFBC1ZAqX9/jL56GEXdLrVe5vT3I22bDVWyDwZX4IEg=", - version = "v0.21.3", - ) - go_repository( - name = "com_google_cloud_go_apigateway", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/apigateway", - sum = "h1:aBSwCQPcp9rZ0zVEUeJbR623palnqtvxJlUyvzsKGQc=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_apigeeconnect", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/apigeeconnect", - sum = "h1:6u/jj0P2c3Mcm+H9qLsXI7gYcTiG9ueyQL3n6vCmFJM=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_apigeeregistry", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/apigeeregistry", - sum = "h1:hgq0ANLDx7t2FDZDJQrCMtCtddR/pjCqVuvQWGrQbXw=", - version = "v0.7.1", - ) - go_repository( - name = "com_google_cloud_go_appengine", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/appengine", - sum = "h1:J+aaUZ6IbTpBegXbmEsh8qZZy864ZVnOoWyfa1XSNbI=", - version = "v1.8.1", - ) - go_repository( - name = "com_google_cloud_go_area120", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/area120", - sum = "h1:wiOq3KDpdqXmaHzvZwKdpoM+3lDcqsI2Lwhyac7stss=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_artifactregistry", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/artifactregistry", - sum = "h1:k6hNqab2CubhWlGcSzunJ7kfxC7UzpAfQ1UPb9PDCKI=", - version = "v1.14.1", - ) - go_repository( - name = "com_google_cloud_go_asset", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/asset", - sum = "h1:vlHdznX70eYW4V1y1PxocvF6tEwxJTTarwIGwOhFF3U=", - version = "v1.14.1", - ) - go_repository( - name = "com_google_cloud_go_assuredworkloads", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/assuredworkloads", - sum = "h1:yaO0kwS+SnhVSTF7BqTyVGt3DTocI6Jqo+S3hHmCwNk=", - version = "v1.11.1", - ) - go_repository( - name = "com_google_cloud_go_automl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/automl", - sum = "h1:iP9iQurb0qbz+YOOMfKSEjhONA/WcoOIjt6/m+6pIgo=", - version = "v1.13.1", - ) - go_repository( - name = "com_google_cloud_go_baremetalsolution", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/baremetalsolution", - sum = "h1:3zztyuQHjfU0C0qEsI9LkC3kf5/TQQ3jUJhbmetUoRA=", - version = "v1.2.0", - ) - go_repository( - name = "com_google_cloud_go_batch", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/batch", - sum = "h1:/4ADpZKoKH300HN2SB6aI7lXX/0hnnbR74wxjLHkyQo=", - version = "v1.4.1", - ) - go_repository( - name = "com_google_cloud_go_beyondcorp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/beyondcorp", - sum = "h1:VPg+fZXULQjs8LiMeWdLaB5oe8G9sEoZ0I0j6IMiG1Q=", - version = "v1.0.0", - ) - go_repository( - name = "com_google_cloud_go_bigquery", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/bigquery", - sum = "h1:hs44Xxov3XLWQiCx2J8lK5U/ihLqnpm4RVVl5fdtLLI=", - version = "v1.55.0", - ) - go_repository( - name = "com_google_cloud_go_billing", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/billing", - sum = "h1:CpagWXb/+QNye+vouomndbc4Gsr0uo+AGR24V16uk8Q=", - version = "v1.17.0", - ) - go_repository( - name = "com_google_cloud_go_binaryauthorization", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/binaryauthorization", - sum = "h1:7L6uUWo/xNCfdVNnnzh2M4x5YA732YPgqRdCG8aKVAU=", - version = "v1.7.0", - ) - go_repository( - name = "com_google_cloud_go_certificatemanager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/certificatemanager", - sum = "h1:uKsohpE0hiobx1Eak9jNcPCznwfB6gvyQCcS28Ah9E8=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_channel", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/channel", - sum = "h1:Hy2EaOiOB7BS1IJmg2lLilEo8uMfFWTy7RgjTzbUqjM=", - version = "v1.17.0", - ) - go_repository( - name = "com_google_cloud_go_cloudbuild", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/cloudbuild", - sum = "h1:YTMxmFra7eIjKFgnyQUxOwWNseNqeO38kGh7thy7v4s=", - version = "v1.14.0", - ) - go_repository( - name = "com_google_cloud_go_clouddms", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/clouddms", - sum = "h1:vTcaFaFZTZZ11gXB6aZHdAx+zn30P8YJw4X/S3NC+VQ=", - version = "v1.7.0", - ) - go_repository( - name = "com_google_cloud_go_cloudtasks", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/cloudtasks", - sum = "h1:cMh9Q6dkvh+Ry5LAPbD/U2aw6KAqdiU6FttwhbTo69w=", - version = "v1.12.1", - ) - go_repository( - name = "com_google_cloud_go_compute", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/compute", - sum = "h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=", - version = "v1.23.0", - ) - go_repository( - name = "com_google_cloud_go_compute_metadata", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/compute/metadata", - sum = "h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=", - version = "v0.2.3", - ) - go_repository( - name = "com_google_cloud_go_contactcenterinsights", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/contactcenterinsights", - sum = "h1:YR2aPedGVQPpFBZXJnPkqRj8M//8veIZZH5ZvICoXnI=", - version = "v1.10.0", - ) - go_repository( - name = "com_google_cloud_go_container", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/container", - sum = "h1:SszQdI0qlyKsImz8/l26rpTZMyqvaH9yfua7rirDZvY=", - version = "v1.26.0", - ) - go_repository( - name = "com_google_cloud_go_containeranalysis", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/containeranalysis", - sum = "h1:/EsoP+UTIjvl4yqrLA4WgUG83kwQhqZmbXEfqirT2LM=", - version = "v0.11.0", - ) - go_repository( - name = "com_google_cloud_go_datacatalog", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/datacatalog", - sum = "h1:qGWrlYvWtK+8jD1jhwq5BsGoSr7S4/LOroV7LwXi00g=", - version = "v1.17.1", - ) - go_repository( - name = "com_google_cloud_go_dataflow", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dataflow", - sum = "h1:VzG2tqsk/HbmOtq/XSfdF4cBvUWRK+S+oL9k4eWkENQ=", - version = "v0.9.1", - ) - go_repository( - name = "com_google_cloud_go_dataform", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dataform", - sum = "h1:xcWso0hKOoxeW72AjBSIp/UfkvpqHNzzS0/oygHlcqY=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_datafusion", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/datafusion", - sum = "h1:eX9CZoyhKQW6g1Xj7+RONeDj1mV8KQDKEB9KLELX9/8=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_datalabeling", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/datalabeling", - sum = "h1:zxsCD/BLKXhNuRssen8lVXChUj8VxF3ofN06JfdWOXw=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_dataplex", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dataplex", - sum = "h1:wqPAP1vRskOoWwNka1yey2wxxCrxRrcxJf78MyFvrbs=", - version = "v1.9.1", - ) - go_repository( - name = "com_google_cloud_go_dataproc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dataproc", - sum = "h1:W47qHL3W4BPkAIbk4SWmIERwsWBaNnWm0P2sdx3YgGU=", - version = "v1.12.0", - ) - go_repository( - name = "com_google_cloud_go_dataproc_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dataproc/v2", - sum = "h1:jKijbdsERm2hy/5dFl/LeQN+7CNssLdGXQYBMvMH/M4=", - version = "v2.2.0", - ) - go_repository( - name = "com_google_cloud_go_dataqna", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dataqna", - sum = "h1:ITpUJep04hC9V7C+gcK390HO++xesQFSUJ7S4nSnF3U=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_datastore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/datastore", - sum = "h1:Mq0ApTRdLW3/dyiw+DkjTk0+iGIUvkbzaC8sfPwWTH4=", - version = "v1.14.0", - ) - go_repository( - name = "com_google_cloud_go_datastream", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/datastream", - sum = "h1:ra/+jMv36zTAGPfi8TRne1hXme+UsKtdcK4j6bnqQiw=", - version = "v1.10.0", - ) - go_repository( - name = "com_google_cloud_go_deploy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/deploy", - sum = "h1:A+w/xpWgz99EYzB6e31gMGAI/P5jTZ2UO7veQK5jQ8o=", - version = "v1.13.0", - ) - go_repository( - name = "com_google_cloud_go_dialogflow", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dialogflow", - sum = "h1:0hBV5ipVbhYNKCyiBoM47bUt+43Kd8eWXhBr+pwUSTw=", - version = "v1.43.0", - ) - go_repository( - name = "com_google_cloud_go_dlp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/dlp", - sum = "h1:tF3wsJ2QulRhRLWPzWVkeDz3FkOGVoMl6cmDUHtfYxw=", - version = "v1.10.1", - ) - go_repository( - name = "com_google_cloud_go_documentai", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/documentai", - sum = "h1:cBndyac7kPWwSuhUcgdbnqzszfZ57HBEHfD33DIwsBM=", - version = "v1.22.1", - ) - go_repository( - name = "com_google_cloud_go_domains", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/domains", - sum = "h1:rqz6KY7mEg7Zs/69U6m6LMbB7PxFDWmT3QWNXIqhHm0=", - version = "v0.9.1", - ) - go_repository( - name = "com_google_cloud_go_edgecontainer", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/edgecontainer", - sum = "h1:zhHWnLzg6AqzE+I3gzJqiIwHfjEBhWctNQEzqb+FaRo=", - version = "v1.1.1", - ) - go_repository( - name = "com_google_cloud_go_errorreporting", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/errorreporting", - sum = "h1:kj1XEWMu8P0qlLhm3FwcaFsUvXChV/OraZwA70trRR0=", - version = "v0.3.0", - ) - go_repository( - name = "com_google_cloud_go_essentialcontacts", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/essentialcontacts", - sum = "h1:OEJ0MLXXCW/tX1fkxzEZOsv/wRfyFsvDVNaHWBAvoV0=", - version = "v1.6.2", - ) - go_repository( - name = "com_google_cloud_go_eventarc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/eventarc", - sum = "h1:xIP3XZi0Xawx8DEfh++mE2lrIi5kQmCr/KcWhJ1q0J4=", - version = "v1.13.0", - ) - go_repository( - name = "com_google_cloud_go_filestore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/filestore", - sum = "h1:Eiz8xZzMJc5ppBWkuaod/PUdUZGCFR8ku0uS+Ah2fRw=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_firestore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/firestore", - sum = "h1:/3S4RssUV4GO/kvgJZB+tayjhOfyAHs+KcpJgRVu/Qk=", - version = "v1.13.0", - ) - go_repository( - name = "com_google_cloud_go_functions", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/functions", - sum = "h1:LtAyqvO1TFmNLcROzHZhV0agEJfBi+zfMZsF4RT/a7U=", - version = "v1.15.1", - ) - go_repository( - name = "com_google_cloud_go_gaming", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/gaming", - sum = "h1:7vEhFnZmd931Mo7sZ6pJy7uQPDxF7m7v8xtBheG08tc=", - version = "v1.9.0", - ) - go_repository( - name = "com_google_cloud_go_gkebackup", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/gkebackup", - sum = "h1:Kfha8SOF2tqsu4O4jVle66mk7qNdlJ2KhL3E2YyiNZc=", - version = "v1.3.1", - ) - go_repository( - name = "com_google_cloud_go_gkeconnect", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/gkeconnect", - sum = "h1:a1ckRvVznnuvDWESM2zZDzSVFvggeBaVY5+BVB8tbT0=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_gkehub", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/gkehub", - sum = "h1:2BLSb8i+Co1P05IYCKATXy5yaaIw/ZqGvVSBTLdzCQo=", - version = "v0.14.1", - ) - go_repository( - name = "com_google_cloud_go_gkemulticloud", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/gkemulticloud", - sum = "h1:MluqhtPVZReoriP5+adGIw+ij/RIeRik8KApCW2WMTw=", - version = "v1.0.0", - ) - go_repository( - name = "com_google_cloud_go_grafeas", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/grafeas", - sum = "h1:oyTL/KjiUeBs9eYLw/40cpSZglUC+0F7X4iu/8t7NWs=", - version = "v0.3.0", - ) - go_repository( - name = "com_google_cloud_go_gsuiteaddons", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/gsuiteaddons", - sum = "h1:mi9jxZpzVjLQibTS/XfPZvl+Jr6D5Bs8pGqUjllRb00=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_iam", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/iam", - sum = "h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4=", - version = "v1.1.2", - ) - go_repository( - name = "com_google_cloud_go_iap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/iap", - sum = "h1:RNhVq/6OMI99/wjPVhqFxjlBxYOBRdaG6rLpBvyaqYY=", - version = "v1.9.0", - ) - go_repository( - name = "com_google_cloud_go_ids", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/ids", - sum = "h1:khXYmSoDDhWGEVxHl4c4IgbwSRR+qE/L4hzP3vaU9Hc=", - version = "v1.4.1", - ) - go_repository( - name = "com_google_cloud_go_iot", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/iot", - sum = "h1:yrH0OSmicD5bqGBoMlWG8UltzdLkYzNUwNVUVz7OT54=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_kms", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/kms", - sum = "h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE=", - version = "v1.15.2", - ) - go_repository( - name = "com_google_cloud_go_language", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/language", - sum = "h1:KnYolG0T5Oex722ZW/sP5QErhVAVNcqpJ16tVJd9RTw=", - version = "v1.11.0", - ) - go_repository( - name = "com_google_cloud_go_lifesciences", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/lifesciences", - sum = "h1:axkANGx1wiBXHiPcJZAE+TDjjYoJRIDzbHC/WYllCBU=", - version = "v0.9.1", - ) - go_repository( - name = "com_google_cloud_go_logging", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/logging", - sum = "h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU=", - version = "v1.8.1", - ) - go_repository( - name = "com_google_cloud_go_longrunning", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/longrunning", - sum = "h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI=", - version = "v0.5.1", - ) - go_repository( - name = "com_google_cloud_go_managedidentities", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/managedidentities", - sum = "h1:2/qZuOeLgUHorSdxSQGtnOu9xQkBn37+j+oZQv/KHJY=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_maps", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/maps", - sum = "h1:PdfgpBLhAoSzZrQXP+/zBc78fIPLZSJp5y8+qSMn2UU=", - version = "v1.4.0", - ) - go_repository( - name = "com_google_cloud_go_mediatranslation", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/mediatranslation", - sum = "h1:50cF7c1l3BanfKrpnTCaTvhf+Fo6kdF21DG0byG7gYU=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_memcache", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/memcache", - sum = "h1:7lkLsF0QF+Mre0O/NvkD9Q5utUNwtzvIYjrOLOs0HO0=", - version = "v1.10.1", - ) - go_repository( - name = "com_google_cloud_go_metastore", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/metastore", - sum = "h1:+9DsxUOHvsqvC0ylrRc/JwzbXJaaBpfIK3tX0Lx8Tcc=", - version = "v1.12.0", - ) - go_repository( - name = "com_google_cloud_go_monitoring", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/monitoring", - sum = "h1:rlndy4K8yknMY9JuGe2aK4SbCh21FXoCdX7SAGHmRgI=", - version = "v1.16.0", - ) - go_repository( - name = "com_google_cloud_go_networkconnectivity", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/networkconnectivity", - sum = "h1:kG2PX6URJ9Kvotfdm+hH8WIhrRY77sAKytUGOz+MgN0=", - version = "v1.13.0", - ) - go_repository( - name = "com_google_cloud_go_networkmanagement", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/networkmanagement", - sum = "h1:aA6L8aioyM4S6nlPYzp2SvB88lBcByZmqMJM6ReafzU=", - version = "v1.9.0", - ) - go_repository( - name = "com_google_cloud_go_networksecurity", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/networksecurity", - sum = "h1:TBLEkMp3AE+6IV/wbIGRNTxnqLXHCTEQWoxRVC18TzY=", - version = "v0.9.1", - ) - go_repository( - name = "com_google_cloud_go_notebooks", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/notebooks", - sum = "h1:6x2K1JAWv6RW2yQO6oa+xtKUGOpGQseCmT94vpOt1vc=", - version = "v1.10.0", - ) - go_repository( - name = "com_google_cloud_go_optimization", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/optimization", - sum = "h1:sGvPVtBJUKNYAwldhJvFmnM+EEdOXjDzjcly3g0n0Xg=", - version = "v1.5.0", - ) - go_repository( - name = "com_google_cloud_go_orchestration", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/orchestration", - sum = "h1:KmN18kE/xa1n91cM5jhCh7s1/UfIguSCisw7nTMUzgE=", - version = "v1.8.1", - ) - go_repository( - name = "com_google_cloud_go_orgpolicy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/orgpolicy", - sum = "h1:I/7dHICQkNwym9erHqmlb50LRU588NPCvkfIY0Bx9jI=", - version = "v1.11.1", - ) - go_repository( - name = "com_google_cloud_go_osconfig", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/osconfig", - sum = "h1:dgyEHdfqML6cUW6/MkihNdTVc0INQst0qSE8Ou1ub9c=", - version = "v1.12.1", - ) - go_repository( - name = "com_google_cloud_go_oslogin", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/oslogin", - sum = "h1:LdSuG3xBYu2Sgr3jTUULL1XCl5QBx6xwzGqzoDUw1j0=", - version = "v1.10.1", - ) - go_repository( - name = "com_google_cloud_go_phishingprotection", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/phishingprotection", - sum = "h1:aK/lNmSd1vtbft/vLe2g7edXK72sIQbqr2QyrZN/iME=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_policytroubleshooter", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/policytroubleshooter", - sum = "h1:pT4qSiL5o0hBSWHDiOcmes/s301PeLLWEhAr/eMQB/g=", - version = "v1.9.0", - ) - go_repository( - name = "com_google_cloud_go_privatecatalog", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/privatecatalog", - sum = "h1:B/18xGo+E0EMS9LOEQ0zXz7F2asMgmVgTYGSI89MHOA=", - version = "v0.9.1", - ) - go_repository( - name = "com_google_cloud_go_pubsub", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/pubsub", - sum = "h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g=", - version = "v1.33.0", - ) - go_repository( - name = "com_google_cloud_go_pubsublite", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/pubsublite", - sum = "h1:pX+idpWMIH30/K7c0epN6V703xpIcMXWRjKJsz0tYGY=", - version = "v1.8.1", - ) - go_repository( - name = "com_google_cloud_go_recaptchaenterprise_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/recaptchaenterprise/v2", - sum = "h1:IGkbudobsTXAwmkEYOzPCQPApUCsN4Gbq3ndGVhHQpI=", - version = "v2.7.2", - ) - go_repository( - name = "com_google_cloud_go_recommendationengine", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/recommendationengine", - sum = "h1:nMr1OEVHuDambRn+/y4RmNAmnR/pXCuHtH0Y4tCgGRQ=", - version = "v0.8.1", - ) - go_repository( - name = "com_google_cloud_go_recommender", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/recommender", - sum = "h1:SuzbMJhDAiPro7tR9QP7EX97+TI31urjsIgNh9XQHl8=", - version = "v1.11.0", - ) - go_repository( - name = "com_google_cloud_go_redis", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/redis", - sum = "h1:YrjQnCC7ydk+k30op7DSjSHw1yAYhqYXFcOq1bSXRYA=", - version = "v1.13.1", - ) - go_repository( - name = "com_google_cloud_go_resourcemanager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/resourcemanager", - sum = "h1:QIAMfndPOHR6yTmMUB0ZN+HSeRmPjR/21Smq5/xwghI=", - version = "v1.9.1", - ) - go_repository( - name = "com_google_cloud_go_resourcesettings", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/resourcesettings", - sum = "h1:Fdyq418U69LhvNPFdlEO29w+DRRjwDA4/pFamm4ksAg=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_retail", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/retail", - sum = "h1:gYBrb9u/Hc5s5lUTFXX1Vsbc/9BEvgtioY6ZKaK0DK8=", - version = "v1.14.1", - ) - go_repository( - name = "com_google_cloud_go_run", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/run", - sum = "h1:kHeIG8q+N6Zv0nDkBjSOYfK2eWqa5FnaiDPH/7/HirE=", - version = "v1.2.0", - ) - go_repository( - name = "com_google_cloud_go_scheduler", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/scheduler", - sum = "h1:yoZbZR8880KgPGLmACOMCiY2tPk+iX4V/dkxqTirlz8=", - version = "v1.10.1", - ) - go_repository( - name = "com_google_cloud_go_secretmanager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/secretmanager", - sum = "h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs=", - version = "v1.11.1", - ) - go_repository( - name = "com_google_cloud_go_security", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/security", - sum = "h1:jR3itwycg/TgGA0uIgTItcVhA55hKWiNJxaNNpQJaZE=", - version = "v1.15.1", - ) - go_repository( - name = "com_google_cloud_go_securitycenter", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/securitycenter", - sum = "h1:XOGJ9OpnDtqg8izd7gYk/XUhj8ytjIalyjjsR6oyG0M=", - version = "v1.23.0", - ) - go_repository( - name = "com_google_cloud_go_servicedirectory", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/servicedirectory", - sum = "h1:pBWpjCFVGWkzVTkqN3TBBIqNSoSHY86/6RL0soSQ4z8=", - version = "v1.11.0", - ) - go_repository( - name = "com_google_cloud_go_shell", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/shell", - sum = "h1:aHbwH9LSqs4r2rbay9f6fKEls61TAjT63jSyglsw7sI=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_spanner", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/spanner", - sum = "h1:+HY8C4uztU7XyLz3xMi/LCXdetLEOExhvRFJu2NiVXM=", - version = "v1.49.0", - ) - go_repository( - name = "com_google_cloud_go_speech", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/speech", - sum = "h1:MCagaq8ObV2tr1kZJcJYgXYbIn8Ai5rp42tyGYw9rls=", - version = "v1.19.0", - ) - go_repository( - name = "com_google_cloud_go_storage", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/storage", - sum = "h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=", - version = "v1.31.0", - ) - go_repository( - name = "com_google_cloud_go_storagetransfer", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/storagetransfer", - sum = "h1:+ZLkeXx0K0Pk5XdDmG0MnUVqIR18lllsihU/yq39I8Q=", - version = "v1.10.0", - ) - go_repository( - name = "com_google_cloud_go_talent", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/talent", - sum = "h1:j46ZgD6N2YdpFPux9mc7OAf4YK3tiBCsbLKc8rQx+bU=", - version = "v1.6.2", - ) - go_repository( - name = "com_google_cloud_go_texttospeech", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/texttospeech", - sum = "h1:S/pR/GZT9p15R7Y2dk2OXD/3AufTct/NSxT4a7nxByw=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_tpu", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/tpu", - sum = "h1:kQf1jgPY04UJBYYjNUO+3GrZtIb57MfGAW2bwgLbR3A=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_trace", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/trace", - sum = "h1:EwGdOLCNfYOOPtgqo+D2sDLZmRCEO1AagRTJCU6ztdg=", - version = "v1.10.1", - ) - go_repository( - name = "com_google_cloud_go_translate", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/translate", - sum = "h1:0na4gC54Lu05ir00dmUSuMkLAojDe1ALq4hBTUkhwjE=", - version = "v1.9.0", - ) - go_repository( - name = "com_google_cloud_go_video", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/video", - sum = "h1:AkjXyJfQ7DtPyDOAbTMeiGcuKsO8/iKSb3fAmTUHYSg=", - version = "v1.20.0", - ) - go_repository( - name = "com_google_cloud_go_videointelligence", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/videointelligence", - sum = "h1:MBMWnkQ78GQnRz5lfdTAbBq/8QMCF3wahgtHh3s/J+k=", - version = "v1.11.1", - ) - go_repository( - name = "com_google_cloud_go_vision_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/vision/v2", - sum = "h1:ccK6/YgPfGHR/CyESz1mvIbsht5Y2xRsWCPqmTNydEw=", - version = "v2.7.2", - ) - go_repository( - name = "com_google_cloud_go_vmmigration", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/vmmigration", - sum = "h1:gnjIclgqbEMc+cF5IJuPxp53wjBIlqZ8h9hE8Rkwp7A=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_vmwareengine", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/vmwareengine", - sum = "h1:qsJ0CPlOQu/3MFBGklu752v3AkD+Pdu091UmXJ+EjTA=", - version = "v1.0.0", - ) - go_repository( - name = "com_google_cloud_go_vpcaccess", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/vpcaccess", - sum = "h1:ram0GzjNWElmbxXMIzeOZUkQ9J8ZAahD6V8ilPGqX0Y=", - version = "v1.7.1", - ) - go_repository( - name = "com_google_cloud_go_webrisk", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/webrisk", - sum = "h1:Ssy3MkOMOnyRV5H2bkMQ13Umv7CwB/kugo3qkAX83Fk=", - version = "v1.9.1", - ) - go_repository( - name = "com_google_cloud_go_websecurityscanner", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/websecurityscanner", - sum = "h1:CfEF/vZ+xXyAR3zC9iaC/QRdf1MEgS20r5UR17Q4gOg=", - version = "v1.6.1", - ) - go_repository( - name = "com_google_cloud_go_workflows", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "cloud.google.com/go/workflows", - sum = "h1:cSUlx4PVV9O0vYCl+pHAUmu0996A7eN602d4wjjVHRs=", - version = "v1.12.0", - ) - go_repository( - name = "com_shuralyov_dmitri_gpu_mtl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "dmitri.shuralyov.com/gpu/mtl", - sum = "h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY=", - version = "v0.0.0-20190408044501-666a987793e9", - ) - go_repository( - name = "com_sslmate_software_src_go_pkcs12", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "software.sslmate.com/src/go-pkcs12", - sum = "h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE=", - version = "v0.2.0", - ) - go_repository( - name = "dev_gocloud", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gocloud.dev", - sum = "h1:PRgA+DXUz8/uuTJDA7wc8o2Hwj9yZ2qAsShZ60esbE8=", - version = "v0.30.0", - ) - go_repository( - name = "in_gopkg_alecthomas_kingpin_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/alecthomas/kingpin.v2", - sum = "h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=", - version = "v2.2.6", - ) - go_repository( - name = "in_gopkg_alexcesaro_statsd_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/alexcesaro/statsd.v2", - sum = "h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc=", - version = "v2.0.0", - ) - go_repository( - name = "in_gopkg_check_v1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/check.v1", - sum = "h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=", - version = "v1.0.0-20201130134442-10cb98267c6c", - ) - go_repository( - name = "in_gopkg_cheggaaa_pb_v1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/cheggaaa/pb.v1", - sum = "h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=", - version = "v1.0.28", - ) - go_repository( - name = "in_gopkg_errgo_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/errgo.v2", - sum = "h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=", - version = "v2.1.0", - ) - go_repository( - name = "in_gopkg_fsnotify_v1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/fsnotify.v1", - sum = "h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=", - version = "v1.4.7", - ) - go_repository( - name = "in_gopkg_gcfg_v1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/gcfg.v1", - sum = "h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs=", - version = "v1.2.3", - ) - go_repository( - name = "in_gopkg_inf_v0", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/inf.v0", - sum = "h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=", - version = "v0.9.1", - ) - go_repository( - name = "in_gopkg_ini_v1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/ini.v1", - sum = "h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=", - version = "v1.67.0", - ) - go_repository( - name = "in_gopkg_natefinch_lumberjack_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/natefinch/lumberjack.v2", - sum = "h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=", - version = "v2.2.1", - ) - go_repository( - name = "in_gopkg_square_go_jose_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/square/go-jose.v2", - sum = "h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=", - version = "v2.6.0", - ) - go_repository( - name = "in_gopkg_src_d_go_billy_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/src-d/go-billy.v4", - sum = "h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=", - version = "v4.3.2", - ) - go_repository( - name = "in_gopkg_src_d_go_git_fixtures_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/src-d/go-git-fixtures.v3", - sum = "h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=", - version = "v3.5.0", - ) - go_repository( - name = "in_gopkg_src_d_go_git_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/src-d/go-git.v4", - sum = "h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=", - version = "v4.13.1", - ) - go_repository( - name = "in_gopkg_tomb_v1", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/tomb.v1", - sum = "h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=", - version = "v1.0.0-20141024135613-dd632973f1e7", - ) - go_repository( - name = "in_gopkg_warnings_v0", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/warnings.v0", - sum = "h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=", - version = "v0.1.2", - ) - go_repository( - name = "in_gopkg_yaml_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/yaml.v2", - sum = "h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=", - version = "v2.4.0", - ) - go_repository( - name = "in_gopkg_yaml_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/yaml.v3", - sum = "h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=", - version = "v3.0.1", - ) - go_repository( - name = "io_etcd_go_bbolt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/bbolt", - sum = "h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=", - version = "v1.3.7", - ) - go_repository( - name = "io_etcd_go_etcd_api_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/api/v3", - sum = "h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs=", - version = "v3.5.9", - ) - go_repository( - name = "io_etcd_go_etcd_client_pkg_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/client/pkg/v3", - sum = "h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE=", - version = "v3.5.9", - ) - go_repository( - name = "io_etcd_go_etcd_client_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/client/v2", - sum = "h1:YZ2OLi0OvR0H75AcgSUajjd5uqKDKocQUqROTG11jIo=", - version = "v2.305.9", - ) - go_repository( - name = "io_etcd_go_etcd_client_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/client/v3", - sum = "h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E=", - version = "v3.5.9", - ) - go_repository( - name = "io_etcd_go_etcd_etcdctl_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/etcdctl/v3", - sum = "h1:2A+/xUck9vBtimGaU8SQh62wCuvuIuREHSGBXBEY6QE=", - version = "v3.5.5", - ) - go_repository( - name = "io_etcd_go_etcd_etcdutl_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/etcdutl/v3", - sum = "h1:KpsQnj71ai24ScrGXF0iwdVZmJU61GK1IbH5oDvYy3M=", - version = "v3.5.5", - ) - go_repository( - name = "io_etcd_go_etcd_pkg_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/pkg/v3", - sum = "h1:6R2jg/aWd/zB9+9JxmijDKStGJAPFsX3e6BeJkMi6eQ=", - version = "v3.5.9", - ) - go_repository( - name = "io_etcd_go_etcd_raft_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/raft/v3", - sum = "h1:ZZ1GIHoUlHsn0QVqiRysAm3/81Xx7+i2d7nSdWxlOiI=", - version = "v3.5.9", - ) - go_repository( - name = "io_etcd_go_etcd_server_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/server/v3", - sum = "h1:vomEmmxeztLtS5OEH7d0hBAg4cjVIu9wXuNzUZx2ZA0=", - version = "v3.5.9", - ) - go_repository( - name = "io_etcd_go_etcd_tests_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/tests/v3", - sum = "h1:QMfo2twT9Erol77/aypdJGN1vtuQ4VNSGnb5cRiIRo8=", - version = "v3.5.5", - ) - go_repository( - name = "io_etcd_go_etcd_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.etcd.io/etcd/v3", - sum = "h1:Dd0pMrzlu2T0FsxDSomE4+8PNxpNJFLKP/cMrZiK/9s=", - version = "v3.5.5", - ) - go_repository( - name = "io_filippo_edwards25519", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "filippo.io/edwards25519", - sum = "h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=", - version = "v1.0.0", - ) - go_repository( - name = "io_k8s_api", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/api", - sum = "h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_apiextensions_apiserver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/apiextensions-apiserver", - sum = "h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_apimachinery", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/apimachinery", - sum = "h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_apiserver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/apiserver", - sum = "h1:rBeYkLvF94Nku9XfXyUIirsVzCzJBs6jMn3NWeHieyI=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_cli_runtime", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/cli-runtime", - sum = "h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_client_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/client-go", - sum = "h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_cloud_provider", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/cloud-provider", - replace = "k8s.io/cloud-provider", - sum = "h1:IiQWyFtdzcPOqvrBZE9FCt0CDCx3GUcZhKkykEgKlM4=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_cluster_bootstrap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/cluster-bootstrap", - sum = "h1:yk1XIWt/mbMgNHFdxd0HyVPq/rnJK7BS3oXj24gHClU=", - version = "v0.27.3", - ) - go_repository( - name = "io_k8s_code_generator", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/code-generator", - sum = "h1:u47guga1rCWLnEnffF09p+cqj8B20oHOLoQ1lb1HGtQ=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_component_base", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/component-base", - sum = "h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_component_helpers", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/component-helpers", - sum = "h1:r/XJ265PMirW9EcGXr/F+2yWrLPo2I69KdvcY/h9HAo=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_controller_manager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/controller-manager", - replace = "k8s.io/controller-manager", - sum = "h1:S7984FVb5ajp8YqMQGAm8zXEUEl0Omw6FJlOiQU2Ne8=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_cri_api", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/cri-api", - sum = "h1:KWO+U8MfI9drXB/P4oU9VchaWYOlwDglJZVHWMpTT3Q=", - version = "v0.27.1", - ) - go_repository( - name = "io_k8s_csi_translation_lib", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/csi-translation-lib", - replace = "k8s.io/csi-translation-lib", - sum = "h1:HbwiOk+M3jIkTC+e5nxUCwmux68OguKV/g9NaHDQhzs=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_dynamic_resource_allocation", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/dynamic-resource-allocation", - replace = "k8s.io/dynamic-resource-allocation", - sum = "h1:lNt4YOVoJqi+wcBesTVJ3KAfr3HnvLedO1/ZovE26pk=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_gengo", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/gengo", - sum = "h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08=", - version = "v0.0.0-20220902162205-c0856e24416d", - ) - go_repository( - name = "io_k8s_klog_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/klog/v2", - sum = "h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=", - version = "v2.100.1", - ) - go_repository( - name = "io_k8s_kms", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kms", - sum = "h1:KhG63LHopCdzs1oKA1j+NWleuIXudgOyCqJo4yi3GaM=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_kube_aggregator", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kube-aggregator", - replace = "k8s.io/kube-aggregator", - sum = "h1:jfHoPip+qN/fn3OcrYs8/xMuVYvkJHKo0H0DYciqdns=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_kube_controller_manager", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kube-controller-manager", - replace = "k8s.io/kube-controller-manager", - sum = "h1:+sPNPN0Fyhycd8iRwpV+zG3eL/uAlekWihgOAZxGZs0=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_kube_openapi", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kube-openapi", - sum = "h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=", - version = "v0.0.0-20230717233707-2695361300d9", - ) - go_repository( - name = "io_k8s_kube_proxy", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kube-proxy", - replace = "k8s.io/kube-proxy", - sum = "h1:nb/ASUpYoXlueURXnY+O2IZkCZmIYOnDprFEeiwwOCY=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_kube_scheduler", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kube-scheduler", - replace = "k8s.io/kube-scheduler", - sum = "h1:ZsN8meIkmJ+wnFrvhi5YzIbueBeBz2xx4I/0cKgpnlg=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_kubectl", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kubectl", - sum = "h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_kubelet", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kubelet", - sum = "h1:5WhTV1iiBu9q/rr+gvy65LQ+K/e7dmgcaYjys5ipLqY=", - version = "v0.27.3", - ) - go_repository( - name = "io_k8s_kubernetes", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/kubernetes", - sum = "h1:K848lTo/D0jvrxUlTvw4nNADixbhXLHgKNDP/KlFGy8=", - version = "v1.27.8", - ) - go_repository( - name = "io_k8s_legacy_cloud_providers", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/legacy-cloud-providers", - replace = "k8s.io/legacy-cloud-providers", - sum = "h1:4D56C4lm+Byu4z34f0sGBkMFlUWpPUqYjaawIrXaGZQ=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_metrics", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/metrics", - sum = "h1:Z/oMk5SmiT/Ji1SaWOPfW2l9W831BLO9/XxDq9iS3ak=", - version = "v0.28.2", - ) - go_repository( - name = "io_k8s_mount_utils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/mount-utils", - sum = "h1:oubkDKLTZUneW27wgyOmp8a1AAZj04vGmtq+YW8wdvY=", - version = "v0.27.3", - ) - go_repository( - name = "io_k8s_pod_security_admission", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/pod-security-admission", - replace = "k8s.io/pod-security-admission", - sum = "h1:dSGK0ftJwJNHSp5fMAwVuFIMMY1MlzW4k82mjar6G8I=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_sample_apiserver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/sample-apiserver", - replace = "k8s.io/sample-apiserver", - sum = "h1:fF3AvtOh/D3HOoIIKC+PIkNHyZJP2uy8Wq/CXiOLXQw=", - version = "v0.27.2", - ) - go_repository( - name = "io_k8s_sigs_apiserver_network_proxy_konnectivity_client", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/apiserver-network-proxy/konnectivity-client", - sum = "h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA=", - version = "v0.1.2", - ) - go_repository( - name = "io_k8s_sigs_controller_runtime", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/controller-runtime", - sum = "h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU=", - version = "v0.15.0", - ) - go_repository( - name = "io_k8s_sigs_json", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/json", - sum = "h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=", - version = "v0.0.0-20221116044647-bc3834ca7abd", - ) - go_repository( - name = "io_k8s_sigs_kustomize_api", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/kustomize/api", - sum = "h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=", - version = "v0.13.5-0.20230601165947-6ce0bf390ce3", - ) - go_repository( - name = "io_k8s_sigs_kustomize_kustomize_v5", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/kustomize/kustomize/v5", - sum = "h1:vq2TtoDcQomhy7OxXLUOzSbHMuMYq0Bjn93cDtJEdKw=", - version = "v5.0.4-0.20230601165947-6ce0bf390ce3", - ) - go_repository( - name = "io_k8s_sigs_kustomize_kyaml", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/kustomize/kyaml", - sum = "h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U=", - version = "v0.14.3-0.20230601165947-6ce0bf390ce3", - ) - go_repository( - name = "io_k8s_sigs_release_utils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/release-utils", - sum = "h1:17LmJrydpUloTCtaoWj95uKlcrUp4h2A9Sa+ZL+lV9w=", - version = "v0.7.4", - ) - go_repository( - name = "io_k8s_sigs_structured_merge_diff_v4", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/structured-merge-diff/v4", - sum = "h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=", - version = "v4.2.3", - ) - go_repository( - name = "io_k8s_sigs_yaml", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "sigs.k8s.io/yaml", - sum = "h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=", - version = "v1.3.0", - ) - go_repository( - name = "io_k8s_system_validators", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/system-validators", - sum = "h1:tq05tdO9zdJZnNF3SXrq6LE7Knc/KfJm5wk68467JDg=", - version = "v1.8.0", - ) - go_repository( - name = "io_k8s_utils", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "k8s.io/utils", - sum = "h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU=", - version = "v0.0.0-20230505201702-9f6742963106", - ) - go_repository( - name = "io_opencensus_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opencensus.io", - sum = "h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=", - version = "v0.24.0", - ) - go_repository( - name = "io_opencensus_go_contrib_exporter_stackdriver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "contrib.go.opencensus.io/exporter/stackdriver", - sum = "h1:bjBKzIf7/TAkxd7L2utGaLM78bmUWlCval5K9UeElbY=", - version = "v0.13.12", - ) - go_repository( - name = "io_opentelemetry_go_contrib_instrumentation_github_com_emicklei_go_restful_otelrestful", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful", - sum = "h1:KQjX0qQ8H21oBUAvFp4ZLKJMMLIluONvSPDAFIGmX58=", - version = "v0.35.0", - ) - go_repository( - name = "io_opentelemetry_go_contrib_instrumentation_google_golang_org_grpc_otelgrpc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc", - sum = "h1:5jD3teb4Qh7mx/nfzq4jO2WFFpvXD0vYWFDrdvNWmXk=", - version = "v0.40.0", - ) - go_repository( - name = "io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp", - sum = "h1:sxoY9kG1s1WpSYNyzm24rlwH4lnRYFXUVVBmKMBfRgw=", - version = "v0.35.1", - ) - go_repository( - name = "io_opentelemetry_go_otel", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel", - sum = "h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_exporters_otlp_internal_retry", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/exporters/otlp/internal/retry", - sum = "h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_exporters_otlp_otlptrace", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/exporters/otlp/otlptrace", - sum = "h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracegrpc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc", - sum = "h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_exporters_otlp_otlptrace_otlptracehttp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp", - sum = "h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_metric", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/metric", - sum = "h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=", - version = "v0.37.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_sdk", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/sdk", - sum = "h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_otel_trace", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/otel/trace", - sum = "h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=", - version = "v1.14.0", - ) - go_repository( - name = "io_opentelemetry_go_proto_otlp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.opentelemetry.io/proto/otlp", - sum = "h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=", - version = "v0.19.0", - ) - go_repository( - name = "io_rsc_binaryregexp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "rsc.io/binaryregexp", - sum = "h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=", - version = "v0.2.0", - ) - go_repository( - name = "io_rsc_quote_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "rsc.io/quote/v3", - sum = "h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY=", - version = "v3.1.0", - ) - go_repository( - name = "io_rsc_sampler", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "rsc.io/sampler", - sum = "h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=", - version = "v1.3.0", - ) - go_repository( - name = "land_oras_oras_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "oras.land/oras-go", - sum = "h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=", - version = "v1.2.4", - ) - go_repository( - name = "net_starlark_go", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.starlark.net", - sum = "h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=", - version = "v0.0.0-20230525235612-a134d8f9ddca", - ) - go_repository( - name = "org_bitbucket_bertimus9_systemstat", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "bitbucket.org/bertimus9/systemstat", - sum = "h1:n0aLnh2Jo4nBUBym9cE5PJDG8GT6g+4VuS2Ya2jYYpA=", - version = "v0.5.0", - ) - go_repository( - name = "org_bitbucket_creachadair_shell", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "bitbucket.org/creachadair/shell", - sum = "h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk=", - version = "v0.0.7", - ) - go_repository( - name = "org_cloudfoundry_code_clock", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "code.cloudfoundry.org/clock", - sum = "h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ=", - version = "v0.0.0-20180518195852-02e53af36e6c", - ) - go_repository( - name = "org_golang_google_api", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/api", - sum = "h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs=", - version = "v0.148.0", - ) - go_repository( - name = "org_golang_google_appengine", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/appengine", - sum = "h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=", - version = "v1.6.7", - ) - go_repository( - name = "org_golang_google_genproto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/genproto", - sum = "h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=", - version = "v0.0.0-20231002182017-d307bd883b97", - ) - go_repository( - name = "org_golang_google_genproto_googleapis_api", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/genproto/googleapis/api", - sum = "h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU=", - version = "v0.0.0-20231002182017-d307bd883b97", - ) - go_repository( - name = "org_golang_google_genproto_googleapis_bytestream", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/genproto/googleapis/bytestream", - sum = "h1:ptblGJZY+SBRhJRVneg4DiHYKwjgeHedTzrgk8CH7m4=", - version = "v0.0.0-20231012201019-e917dd12ba7a", - ) - go_repository( - name = "org_golang_google_genproto_googleapis_rpc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/genproto/googleapis/rpc", - sum = "h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c=", - version = "v0.0.0-20231012201019-e917dd12ba7a", - ) - go_repository( - name = "org_golang_google_grpc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/grpc", - sum = "h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=", - version = "v1.59.0", - ) - go_repository( - name = "org_golang_google_grpc_cmd_protoc_gen_go_grpc", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/grpc/cmd/protoc-gen-go-grpc", - sum = "h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE=", - version = "v1.1.0", - ) - go_repository( - name = "org_golang_google_protobuf", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "google.golang.org/protobuf", - sum = "h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=", - version = "v1.31.0", - ) - go_repository( - name = "org_golang_x_crypto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/crypto", - sum = "h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=", - version = "v0.14.0", - ) - go_repository( - name = "org_golang_x_exp", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/exp", - sum = "h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U=", - version = "v0.0.0-20230809150735-7b3493d9a819", - ) - go_repository( - name = "org_golang_x_image", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/image", - sum = "h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=", - version = "v0.0.0-20190802002840-cff245a6509b", - ) - go_repository( - name = "org_golang_x_lint", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/lint", - sum = "h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=", - version = "v0.0.0-20210508222113-6edffad5e616", - ) - go_repository( - name = "org_golang_x_mobile", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/mobile", - sum = "h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=", - version = "v0.0.0-20190719004257-d2bd2a29d028", - ) - go_repository( - name = "org_golang_x_mod", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/mod", - sum = "h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=", - version = "v0.13.0", - ) - go_repository( - name = "org_golang_x_net", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/net", - sum = "h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=", - version = "v0.17.0", - ) - go_repository( - name = "org_golang_x_oauth2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/oauth2", - sum = "h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=", - version = "v0.13.0", - ) - go_repository( - name = "org_golang_x_sync", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/sync", - sum = "h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=", - version = "v0.4.0", - ) - go_repository( - name = "org_golang_x_sys", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/sys", - sum = "h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=", - version = "v0.13.0", - ) - go_repository( - name = "org_golang_x_term", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/term", - sum = "h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=", - version = "v0.13.0", - ) - go_repository( - name = "org_golang_x_text", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/text", - sum = "h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=", - version = "v0.14.0", - ) - go_repository( - name = "org_golang_x_time", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/time", - sum = "h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=", - version = "v0.3.0", - ) - go_repository( - name = "org_golang_x_tools", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/tools", - sum = "h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=", - version = "v0.14.0", - ) - go_repository( - name = "org_golang_x_vuln", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/vuln", - sum = "h1:KUas02EjQK5LTuIx1OylBQdKKZ9jeugs+HiqO5HormU=", - version = "v1.0.1", - ) - go_repository( - name = "org_golang_x_xerrors", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "golang.org/x/xerrors", - sum = "h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=", - version = "v0.0.0-20220907171357-04be3eba64a2", - ) - go_repository( - name = "org_libvirt_go_libvirt", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "libvirt.org/go/libvirt", - # keep - patches = [ - "//3rdparty/bazel/org_libvirt_go_libvirt:go_libvirt.patch", - ], - sum = "h1:u+CHhs2OhVmu0MWzBDrlbLzQ5QB3ZfWtfT+lD3EaUIs=", - version = "v1.9004.0", - ) - go_repository( - name = "org_mongodb_go_mongo_driver", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.mongodb.org/mongo-driver", - sum = "h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=", - version = "v1.11.3", - ) - go_repository( - name = "org_mozilla_go_pkcs7", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.mozilla.org/pkcs7", - sum = "h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M=", - version = "v0.0.0-20200128120323-432b2356ecb1", - ) - go_repository( - name = "org_uber_go_atomic", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.uber.org/atomic", - sum = "h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=", - version = "v1.11.0", - ) - go_repository( - name = "org_uber_go_goleak", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.uber.org/goleak", - sum = "h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=", - version = "v1.3.0", - ) - go_repository( - name = "org_uber_go_multierr", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.uber.org/multierr", - sum = "h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=", - version = "v1.11.0", - ) - go_repository( - name = "org_uber_go_zap", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.uber.org/zap", - sum = "h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=", - version = "v1.26.0", - ) - go_repository( - name = "sh_helm_helm", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "helm.sh/helm", - sum = "h1:cSe3FaQOpRWLDXvTObQNj0P7WI98IG5yloU6tQVls2k=", - version = "v2.17.0+incompatible", - ) - go_repository( - name = "sh_helm_helm_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "helm.sh/helm/v3", - sum = "h1:DG+XLGzBJeZvMLlMbm6bPDLV1dGaVW9eZsDoUd1/LM0=", - version = "v3.13.1", - ) - go_repository( - name = "sm_step_go_crypto", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "go.step.sm/crypto", - sum = "h1:EhJpFRNgU3RaNEO3WZ62Kn2gF9NWNglNG4DvSPeuiTs=", - version = "v0.32.2", - ) - go_repository( - name = "tools_gotest_v3", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gotest.tools/v3", - sum = "h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=", - version = "v3.4.0", - ) - go_repository( - name = "xyz_gomodules_jsonpatch_v2", - build_file_generation = "on", - build_file_proto_mode = "disable_global", - importpath = "gomodules.xyz/jsonpatch/v2", - sum = "h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc=", - version = "v2.3.0", - ) diff --git a/bazel/toolchains/go_rules_deps.bzl b/bazel/toolchains/go_rules_deps.bzl deleted file mode 100644 index f76d7eed0..000000000 --- a/bazel/toolchains/go_rules_deps.bzl +++ /dev/null @@ -1,29 +0,0 @@ -"""Go toolchain dependencies for Bazel. - -Defines hermetic go toolchains and rules to build and test go code. -Gazelle is a build file generator for Bazel projects written in Go. -""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def go_deps(): - http_archive( - name = "io_bazel_rules_go", - sha256 = "91585017debb61982f7054c9688857a2ad1fd823fc3f9cb05048b0025c47d023", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip", - "https://cdn.confidential.cloud/constellation/cas/sha256/91585017debb61982f7054c9688857a2ad1fd823fc3f9cb05048b0025c47d023", - "https://github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip", - ], - type = "zip", - ) - http_archive( - name = "bazel_gazelle", - sha256 = "b7387f72efb59f876e4daae42f1d3912d0d45563eac7cb23d1de0b094ab588cf", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz", - "https://cdn.confidential.cloud/constellation/cas/sha256/b7387f72efb59f876e4daae42f1d3912d0d45563eac7cb23d1de0b094ab588cf", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.34.0/bazel-gazelle-v0.34.0.tar.gz", - ], - type = "tar.gz", - ) diff --git a/bazel/toolchains/hermetic_cc_deps.bzl b/bazel/toolchains/hermetic_cc_deps.bzl deleted file mode 100644 index 2ace3b68a..000000000 --- a/bazel/toolchains/hermetic_cc_deps.bzl +++ /dev/null @@ -1,16 +0,0 @@ -"""hermetic cc toolchain (bazel-zig-cc) build rules""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def hermetic_cc_deps(): - """Loads the dependencies for hermetic_cc_toolchain.""" - - http_archive( - name = "hermetic_cc_toolchain", - urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/a5caccbf6d86d4f60afd45b541a05ca4cc3f5f523aec7d3f7711e584600fb075", - "https://github.com/uber/hermetic_cc_toolchain/releases/download/v2.1.3/hermetic_cc_toolchain-v2.1.3.tar.gz", - ], - type = "tar.gz", - sha256 = "a5caccbf6d86d4f60afd45b541a05ca4cc3f5f523aec7d3f7711e584600fb075", - ) diff --git a/bazel/toolchains/k8s.bzl b/bazel/toolchains/k8s.bzl index 229c23667..3256870ee 100644 --- a/bazel/toolchains/k8s.bzl +++ b/bazel/toolchains/k8s.bzl @@ -11,10 +11,10 @@ def envtest_deps(): strip_prefix = "kubebuilder/bin", type = "tar.gz", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/e1913674bacaa70c067e15649237e1f67d891ba53f367c0a50786b4a274ee047", - "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.27.1-darwin-amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/b62112fc83e503b35eab4fa5f8f5f648fcc4781ec319e6844644b4502eb8e2f1", + "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.29.1-darwin-amd64.tar.gz", ], - sha256 = "e1913674bacaa70c067e15649237e1f67d891ba53f367c0a50786b4a274ee047", + sha256 = "b62112fc83e503b35eab4fa5f8f5f648fcc4781ec319e6844644b4502eb8e2f1", build_file_content = """exports_files(["etcd", "kubectl", "kube-apiserver"], visibility = ["//visibility:public"])""", ) http_archive( @@ -22,10 +22,10 @@ def envtest_deps(): strip_prefix = "kubebuilder/bin", type = "tar.gz", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/0422632a2bbb0d4d14d7d8b0f05497a4d041c11d770a07b7a55c44bcc5e8ce66", - "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.27.1-darwin-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/61b0eea4a7095cf0c318bd35f8c16b8f5107c5af65a710abf5a95f9ed29fd593", + "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.29.1-darwin-arm64.tar.gz", ], - sha256 = "0422632a2bbb0d4d14d7d8b0f05497a4d041c11d770a07b7a55c44bcc5e8ce66", + sha256 = "61b0eea4a7095cf0c318bd35f8c16b8f5107c5af65a710abf5a95f9ed29fd593", build_file_content = """exports_files(["etcd", "kubectl", "kube-apiserver"], visibility = ["//visibility:public"])""", ) http_archive( @@ -33,10 +33,10 @@ def envtest_deps(): strip_prefix = "kubebuilder/bin", type = "tar.gz", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/f9699df7b021f71a1ab55329b36b48a798e6ae3a44d2132255fc7e46c6790d4d", - "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.27.1-linux-amd64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/e37c2cda216692e699ce40ac2067dac8d773654f4afb20a90b92e6aafbe1593c", + "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.29.1-linux-amd64.tar.gz", ], - sha256 = "f9699df7b021f71a1ab55329b36b48a798e6ae3a44d2132255fc7e46c6790d4d", + sha256 = "e37c2cda216692e699ce40ac2067dac8d773654f4afb20a90b92e6aafbe1593c", build_file_content = """exports_files(["etcd", "kubectl", "kube-apiserver"], visibility = ["//visibility:public"])""", ) http_archive( @@ -44,9 +44,9 @@ def envtest_deps(): strip_prefix = "kubebuilder/bin", type = "tar.gz", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/9d2803e8ca85c465b33c12b06d0b2eba3ddb64b53a468628f741e50b462c46ad", - "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.27.1-linux-arm64.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/956783315fde4371e775b1c7f115c996494b8603b3bf75cb528b0df9b6536382", + "https://kubebuilder-tools.storage.googleapis.com/kubebuilder-tools-1.29.1-linux-arm64.tar.gz", ], - sha256 = "9d2803e8ca85c465b33c12b06d0b2eba3ddb64b53a468628f741e50b462c46ad", + sha256 = "956783315fde4371e775b1c7f115c996494b8603b3bf75cb528b0df9b6536382", build_file_content = """exports_files(["etcd", "kubectl", "kube-apiserver"], visibility = ["//visibility:public"])""", ) diff --git a/bazel/toolchains/linux_kernel.bzl b/bazel/toolchains/linux_kernel.bzl index 0c0e95a3b..d34d15324 100644 --- a/bazel/toolchains/linux_kernel.bzl +++ b/bazel/toolchains/linux_kernel.bzl @@ -3,39 +3,80 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") def kernel_rpms(): + """kernel rpms""" + + # LTS kernel http_file( - name = "kernel", + name = "kernel_lts", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/4430d2f8076081291d505ccb91bc84e3a763e113348e23775cc01df5a574d684", - "https://cdn.confidential.cloud/constellation/kernel/6.1.68-100.constellation/kernel-6.1.68-100.constellation.fc38.x86_64.rpm", + "https://cdn.confidential.cloud/constellation/cas/sha256/7834bc4bc7e088c98505956382884bdc670ab9a9283288b7fef04a43df31356e", + "https://cdn.confidential.cloud/constellation/kernel/6.6.87-100.constellation/kernel-6.6.87-100.constellation.fc40.x86_64.rpm", ], - downloaded_file_path = "kernel.rpm", - sha256 = "4430d2f8076081291d505ccb91bc84e3a763e113348e23775cc01df5a574d684", + downloaded_file_path = "kernel-lts.rpm", + sha256 = "7834bc4bc7e088c98505956382884bdc670ab9a9283288b7fef04a43df31356e", ) http_file( - name = "kernel_core", + name = "kernel_core_lts", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/e3f9a42c4c86d56cae98053d3fc099368cbcf6dfa8ed48848e24e2c82ae3b7cc", - "https://cdn.confidential.cloud/constellation/kernel/6.1.68-100.constellation/kernel-core-6.1.68-100.constellation.fc38.x86_64.rpm", + "https://cdn.confidential.cloud/constellation/cas/sha256/2763c699d1e2f9810421ac7af2e9c94c6f98533e83f2938c26f1d824e3559b97", + "https://cdn.confidential.cloud/constellation/kernel/6.6.87-100.constellation/kernel-core-6.6.87-100.constellation.fc40.x86_64.rpm", ], - downloaded_file_path = "kernel-core.rpm", - sha256 = "e3f9a42c4c86d56cae98053d3fc099368cbcf6dfa8ed48848e24e2c82ae3b7cc", + downloaded_file_path = "kernel-core-lts.rpm", + sha256 = "2763c699d1e2f9810421ac7af2e9c94c6f98533e83f2938c26f1d824e3559b97", ) http_file( - name = "kernel_modules", + name = "kernel_modules_lts", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/072fc8e1b1bb37e1cc40038f60e21a7be374d801f48589146660ffe7028f6b39", - "https://cdn.confidential.cloud/constellation/kernel/6.1.68-100.constellation/kernel-modules-6.1.68-100.constellation.fc38.x86_64.rpm", + "https://cdn.confidential.cloud/constellation/cas/sha256/a7604eec263f190db573d809d20336bbf75e46c51f5977f5db95bb88bfec56d3", + "https://cdn.confidential.cloud/constellation/kernel/6.6.87-100.constellation/kernel-modules-6.6.87-100.constellation.fc40.x86_64.rpm", ], - downloaded_file_path = "kernel-modules.rpm", - sha256 = "072fc8e1b1bb37e1cc40038f60e21a7be374d801f48589146660ffe7028f6b39", + downloaded_file_path = "kernel-modules-lts.rpm", + sha256 = "a7604eec263f190db573d809d20336bbf75e46c51f5977f5db95bb88bfec56d3", ) http_file( - name = "kernel_modules_core", + name = "kernel_modules_core_lts", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/97d1e099b874d53b87fad2515c450b33d56770236211bf6a83a52e9e28361be1", - "https://cdn.confidential.cloud/constellation/kernel/6.1.68-100.constellation/kernel-modules-core-6.1.68-100.constellation.fc38.x86_64.rpm", + "https://cdn.confidential.cloud/constellation/cas/sha256/648fd503d7d54608fbd62ace87c4da098f72abbaac1ab7e343327fc24ccef7f8", + "https://cdn.confidential.cloud/constellation/kernel/6.6.87-100.constellation/kernel-modules-core-6.6.87-100.constellation.fc40.x86_64.rpm", ], - downloaded_file_path = "kernel-modules-core.rpm", - sha256 = "97d1e099b874d53b87fad2515c450b33d56770236211bf6a83a52e9e28361be1", + downloaded_file_path = "kernel-modules-core-lts.rpm", + sha256 = "648fd503d7d54608fbd62ace87c4da098f72abbaac1ab7e343327fc24ccef7f8", + ) + + # mainline kernel + http_file( + name = "kernel_mainline", + urls = [ + "https://cdn.confidential.cloud/constellation/cas/sha256/6eaec29870e6549d95a93b72ea10715507db84b851c68c0d75e44e4c20f895f2", + "https://kojipkgs.fedoraproject.org/packages/kernel/6.8.9/300.fc40/x86_64/kernel-6.8.9-300.fc40.x86_64.rpm", + ], + downloaded_file_path = "kernel-mainline.rpm", + sha256 = "6eaec29870e6549d95a93b72ea10715507db84b851c68c0d75e44e4c20f895f2", + ) + http_file( + name = "kernel_core_mainline", + urls = [ + "https://cdn.confidential.cloud/constellation/cas/sha256/910fd35209f7dc8185e88dddeaccf6158dd63ad9fd469ef3dc81b96840ef28eb", + "https://kojipkgs.fedoraproject.org/packages/kernel/6.8.9/300.fc40/x86_64/kernel-core-6.8.9-300.fc40.x86_64.rpm", + ], + downloaded_file_path = "kernel-core-mainline.rpm", + sha256 = "910fd35209f7dc8185e88dddeaccf6158dd63ad9fd469ef3dc81b96840ef28eb", + ) + http_file( + name = "kernel_modules_mainline", + urls = [ + "https://cdn.confidential.cloud/constellation/cas/sha256/b8de20433c68d2fe0ca6625e25f314aba36a9327592db8b1478b97bb50521149", + "https://kojipkgs.fedoraproject.org/packages/kernel/6.8.9/300.fc40/x86_64/kernel-modules-6.8.9-300.fc40.x86_64.rpm", + ], + downloaded_file_path = "kernel-modules-mainline.rpm", + sha256 = "b8de20433c68d2fe0ca6625e25f314aba36a9327592db8b1478b97bb50521149", + ) + http_file( + name = "kernel_modules_core_mainline", + urls = [ + "https://cdn.confidential.cloud/constellation/cas/sha256/8ecd8e96483810d18e04a20cd8ecef46f27bff0fbb54f23e67adb813828b3cec", + "https://kojipkgs.fedoraproject.org/packages/kernel/6.8.9/300.fc40/x86_64/kernel-modules-core-6.8.9-300.fc40.x86_64.rpm", + ], + downloaded_file_path = "kernel-modules-core-mainline.rpm", + sha256 = "8ecd8e96483810d18e04a20cd8ecef46f27bff0fbb54f23e67adb813828b3cec", ) diff --git a/bazel/toolchains/multirun_deps.bzl b/bazel/toolchains/multirun_deps.bzl index b34da1cec..ee1f3a584 100644 --- a/bazel/toolchains/multirun_deps.bzl +++ b/bazel/toolchains/multirun_deps.bzl @@ -5,11 +5,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def multirun_deps(): http_archive( name = "com_github_ash2k_bazel_tools", - sha256 = "a911dab6711bc12a00f02cc94b66ced7dc57650e382ebd4f17c9cdb8ec2cbd56", - strip_prefix = "bazel-tools-2add5bb84c2837a82a44b57e83c7414247aed43a", + sha256 = "dc32a65c69c843f1ba2a328b79974163896e5b8ed283cd711abe12bf7cd12ffc", + strip_prefix = "bazel-tools-415483a9e13342a6603a710b0296f6d85b8d26bf", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/a911dab6711bc12a00f02cc94b66ced7dc57650e382ebd4f17c9cdb8ec2cbd56", - "https://github.com/ash2k/bazel-tools/archive/2add5bb84c2837a82a44b57e83c7414247aed43a.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/dc32a65c69c843f1ba2a328b79974163896e5b8ed283cd711abe12bf7cd12ffc", + "https://github.com/ash2k/bazel-tools/archive/415483a9e13342a6603a710b0296f6d85b8d26bf.tar.gz", ], type = "tar.gz", ) diff --git a/bazel/toolchains/nixpkgs_deps.bzl b/bazel/toolchains/nixpkgs_deps.bzl index 350b64d1d..bdf5f0f1c 100644 --- a/bazel/toolchains/nixpkgs_deps.bzl +++ b/bazel/toolchains/nixpkgs_deps.bzl @@ -5,11 +5,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def nixpkgs_deps(): http_archive( name = "io_tweag_rules_nixpkgs", - sha256 = "cf84628af3e4698acb200c005c4acf1dddaf5e7b9f839eeca78d983db2e874fb", - strip_prefix = "rules_nixpkgs-2c767691d12b66a92f231bccb06bcf9f7477b962", + sha256 = "30271f7bd380e4e20e4d7132c324946c4fdbc31ebe0bbb6638a0f61a37e74397", + strip_prefix = "rules_nixpkgs-0.13.0", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/cf84628af3e4698acb200c005c4acf1dddaf5e7b9f839eeca78d983db2e874fb", - "https://github.com/tweag/rules_nixpkgs/archive/2c767691d12b66a92f231bccb06bcf9f7477b962.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/30271f7bd380e4e20e4d7132c324946c4fdbc31ebe0bbb6638a0f61a37e74397", + "https://github.com/tweag/rules_nixpkgs/releases/download/v0.13.0/rules_nixpkgs-0.13.0.tar.gz", ], type = "tar.gz", ) diff --git a/bazel/toolchains/oci_deps.bzl b/bazel/toolchains/oci_deps.bzl index 77f13f2b7..f425e9066 100644 --- a/bazel/toolchains/oci_deps.bzl +++ b/bazel/toolchains/oci_deps.bzl @@ -3,13 +3,17 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def oci_deps(): + # TODO(malt3): This uses a patch on top of the normal rules_oci that removes broken Windows support. + # Remove this override once https://github.com/bazel-contrib/rules_oci/issues/420 is fixed. http_archive( name = "rules_oci", - strip_prefix = "rules_oci-0.4.0", + strip_prefix = "rules_oci-2.2.5", type = "tar.gz", urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/d7b0760ba28554b71941ea0bbfd0a9f089bf250fd4448f9c116e1cb7a63b3933", - "https://github.com/bazel-contrib/rules_oci/releases/download/v0.4.0/rules_oci-v0.4.0.tar.gz", + "https://cdn.confidential.cloud/constellation/cas/sha256/361c417e8c95cd7c3d8b5cf4b202e76bac8d41532131534ff8e6fa43aa161142", + "https://github.com/bazel-contrib/rules_oci/releases/download/v2.2.5/rules_oci-v2.2.5.tar.gz", ], - sha256 = "d7b0760ba28554b71941ea0bbfd0a9f089bf250fd4448f9c116e1cb7a63b3933", + sha256 = "361c417e8c95cd7c3d8b5cf4b202e76bac8d41532131534ff8e6fa43aa161142", + patches = ["//bazel/toolchains:0001-disable-Windows-support.patch"], + patch_args = ["-p1"], ) diff --git a/bazel/toolchains/pkg_deps.bzl b/bazel/toolchains/pkg_deps.bzl deleted file mode 100644 index ff734a80d..000000000 --- a/bazel/toolchains/pkg_deps.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""rules_pkg dependencies""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def pkg_deps(): - http_archive( - name = "rules_pkg", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", - "https://cdn.confidential.cloud/constellation/cas/sha256/8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", - ], - sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", - type = "tar.gz", - ) diff --git a/bazel/toolchains/proto_deps.bzl b/bazel/toolchains/proto_deps.bzl deleted file mode 100644 index 72b02ff09..000000000 --- a/bazel/toolchains/proto_deps.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""proto toolchain rules""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def proto_deps(): - http_archive( - name = "rules_proto", - sha256 = "17fa03f509b0d1df05c70c174a266ab211d04b9969e41924fd07a81ea171f117", - strip_prefix = "rules_proto-cda0effe6b5af095a6886c67f90c760b83f08c48", - urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/17fa03f509b0d1df05c70c174a266ab211d04b9969e41924fd07a81ea171f117", - "https://github.com/bazelbuild/rules_proto/archive/cda0effe6b5af095a6886c67f90c760b83f08c48.tar.gz", - ], - type = "tar.gz", - ) diff --git a/bazel/toolchains/python_deps.bzl b/bazel/toolchains/python_deps.bzl deleted file mode 100644 index 1149cb5c4..000000000 --- a/bazel/toolchains/python_deps.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""python toolchain rules""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def python_deps(): - http_archive( - name = "rules_python", - strip_prefix = "rules_python-0.25.0", - urls = [ - "https://cdn.confidential.cloud/constellation/cas/sha256/5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036", - "https://github.com/bazelbuild/rules_python/releases/download/0.25.0/rules_python-0.25.0.tar.gz", - ], - type = "tar.gz", - sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036", - ) diff --git a/bazel/toolchains/skylib_deps.bzl b/bazel/toolchains/skylib_deps.bzl deleted file mode 100644 index c86bfe9e1..000000000 --- a/bazel/toolchains/skylib_deps.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""bazel skylib""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def skylib_deps(): - http_archive( - name = "bazel_skylib", - sha256 = "cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", - "https://cdn.confidential.cloud/constellation/cas/sha256/cd55a062e763b9349921f0f5db8c3933288dc8ba4f76dd9416aac68acee3cb94", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.5.0/bazel-skylib-1.5.0.tar.gz", - ], - type = "tar.gz", - ) diff --git a/bootstrapper/cmd/bootstrapper/BUILD.bazel b/bootstrapper/cmd/bootstrapper/BUILD.bazel index 628fea394..77896efe7 100644 --- a/bootstrapper/cmd/bootstrapper/BUILD.bazel +++ b/bootstrapper/cmd/bootstrapper/BUILD.bazel @@ -20,8 +20,8 @@ go_library( "//bootstrapper/internal/kubernetes", "//bootstrapper/internal/kubernetes/k8sapi", "//bootstrapper/internal/kubernetes/kubewaiter", - "//bootstrapper/internal/logging", "//bootstrapper/internal/nodelock", + "//bootstrapper/internal/reboot", "//internal/atls", "//internal/attestation/choose", "//internal/attestation/initialize", @@ -45,7 +45,6 @@ go_library( "//internal/versions/components", "@com_github_spf13_afero//:afero", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", - "@org_uber_go_zap//:zap", ], ) diff --git a/bootstrapper/cmd/bootstrapper/main.go b/bootstrapper/cmd/bootstrapper/main.go index f2f600823..f1a533245 100644 --- a/bootstrapper/cmd/bootstrapper/main.go +++ b/bootstrapper/cmd/bootstrapper/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -9,17 +9,17 @@ package main import ( "context" "flag" + "fmt" "io" + "log/slog" "os" "strconv" "github.com/spf13/afero" - "go.uber.org/zap" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/logging" "github.com/edgelesssys/constellation/v2/internal/attestation/choose" "github.com/edgelesssys/constellation/v2/internal/attestation/simulator" "github.com/edgelesssys/constellation/v2/internal/attestation/tdx" @@ -43,17 +43,10 @@ const ( ) func main() { - gRPCDebug := flag.Bool("debug", false, "Enable gRPC debug logging") verbosity := flag.Int("v", 0, logger.CmdLineVerbosityDescription) flag.Parse() - log := logger.New(logger.JSONLog, logger.VerbosityFromInt(*verbosity)).Named("bootstrapper") - defer log.Sync() - - if *gRPCDebug { - log.Named("gRPC").ReplaceGRPCLogger() - } else { - log.Named("gRPC").WithIncreasedLevel(zap.WarnLevel).ReplaceGRPCLogger() - } + log := logger.NewJSONLogger(logger.VerbosityFromInt(*verbosity)).WithGroup("bootstrapper") + logger.ReplaceGRPCLogger(logger.GRPCLogger(log)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -62,35 +55,32 @@ func main() { bindPort := strconv.Itoa(constants.BootstrapperPort) var clusterInitJoiner clusterInitJoiner var metadataAPI metadataAPI - var cloudLogger logging.CloudLogger var openDevice vtpm.TPMOpenFunc var fs afero.Fs attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant)) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant") + log.With(slog.Any("error", err)).Error("Failed to parse attestation variant") + os.Exit(1) } issuer, err := choose.Issuer(attestVariant, log) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to select issuer") + log.With(slog.Any("error", err)).Error("Failed to select issuer") + os.Exit(1) } switch cloudprovider.FromString(os.Getenv(constellationCSP)) { case cloudprovider.AWS: metadata, err := awscloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to set up AWS metadata API") + log.With(slog.Any("error", err)).Error("Failed to set up AWS metadata API") + os.Exit(1) } metadataAPI = metadata - cloudLogger, err = awscloud.NewLogger(ctx) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") - } - clusterInitJoiner = kubernetes.New( "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.NewUninitialized(), - metadata, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, log, ) openDevice = vtpm.OpenVTPM fs = afero.NewOsFs() @@ -98,52 +88,45 @@ func main() { case cloudprovider.GCP: metadata, err := gcpcloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create GCP metadata client") + log.With(slog.Any("error", err)).Error("Failed to create GCP metadata client") + os.Exit(1) } defer metadata.Close() - cloudLogger, err = gcpcloud.NewLogger(ctx, "constellation-boot-log") - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") - } - metadataAPI = metadata clusterInitJoiner = kubernetes.New( "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.NewUninitialized(), - metadata, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, log, ) openDevice = vtpm.OpenVTPM fs = afero.NewOsFs() - log.Infof("Added load balancer IP to routing table") case cloudprovider.Azure: metadata, err := azurecloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create Azure metadata client") - } - cloudLogger, err = azurecloud.NewLogger(ctx) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to set up cloud logger") + log.With(slog.Any("error", err)).Error("Failed to create Azure metadata client") + os.Exit(1) } + if err := metadata.PrepareControlPlaneNode(ctx, log); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to prepare Azure control plane node") + log.With(slog.Any("error", err)).Error("Failed to prepare Azure control plane node") + os.Exit(1) } metadataAPI = metadata clusterInitJoiner = kubernetes.New( "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.NewUninitialized(), - metadata, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, log, ) openDevice = vtpm.OpenVTPM fs = afero.NewOsFs() case cloudprovider.QEMU: - cloudLogger = qemucloud.NewLogger() metadata := qemucloud.New() clusterInitJoiner = kubernetes.New( "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.NewUninitialized(), - metadata, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, log, ) metadataAPI = metadata @@ -155,18 +138,18 @@ func main() { return tdx.Open() } default: - log.Fatalf("Unsupported attestation variant: %s", attestVariant) + log.Error(fmt.Sprintf("Unsupported attestation variant: %s", attestVariant)) } fs = afero.NewOsFs() case cloudprovider.OpenStack: - cloudLogger = &logging.NopLogger{} metadata, err := openstackcloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create OpenStack metadata client") + log.With(slog.Any("error", err)).Error("Failed to create OpenStack metadata client") + os.Exit(1) } clusterInitJoiner = kubernetes.New( "openstack", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.NewUninitialized(), - metadata, &kubewaiter.CloudKubeAPIWaiter{}, + metadata, &kubewaiter.CloudKubeAPIWaiter{}, log, ) metadataAPI = metadata openDevice = vtpm.OpenVTPM @@ -174,7 +157,6 @@ func main() { default: clusterInitJoiner = &clusterFake{} metadataAPI = &providerMetadataFake{} - cloudLogger = &logging.NopLogger{} var simulatedTPMCloser io.Closer openDevice, simulatedTPMCloser = simulator.NewSimulatedTPMOpenFunc() defer simulatedTPMCloser.Close() @@ -183,5 +165,5 @@ func main() { fileHandler := file.NewHandler(fs) - run(issuer, openDevice, fileHandler, clusterInitJoiner, metadataAPI, bindIP, bindPort, log, cloudLogger) + run(issuer, openDevice, fileHandler, clusterInitJoiner, metadataAPI, bindIP, bindPort, log) } diff --git a/bootstrapper/cmd/bootstrapper/run.go b/bootstrapper/cmd/bootstrapper/run.go index c036b5ba9..fdb5891a6 100644 --- a/bootstrapper/cmd/bootstrapper/run.go +++ b/bootstrapper/cmd/bootstrapper/run.go @@ -1,87 +1,101 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main import ( "context" + "fmt" + "log/slog" "net" + "sync" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/clean" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/initserver" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/joinclient" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/logging" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/nodelock" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/reboot" "github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/attestation/initialize" "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" - "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" ) func run(issuer atls.Issuer, openDevice vtpm.TPMOpenFunc, fileHandler file.Handler, kube clusterInitJoiner, metadata metadataAPI, - bindIP, bindPort string, log *logger.Logger, - cloudLogger logging.CloudLogger, + bindIP, bindPort string, log *slog.Logger, ) { - defer cloudLogger.Close() + log.With(slog.String("version", constants.BinaryVersion().String())).Info("Starting bootstrapper") - log.With(zap.String("version", constants.BinaryVersion().String())).Infof("Starting bootstrapper") - cloudLogger.Disclose("bootstrapper started running...") - - uuid, err := getDiskUUID() + disk := diskencryption.New() + uuid, err := getDiskUUID(disk) if err != nil { - log.With(zap.Error(err)).Errorf("Failed to get disk UUID") - cloudLogger.Disclose("Failed to get disk UUID") + log.With(slog.Any("error", err)).Error("Failed to get disk UUID") } else { - log.Infof("Disk UUID: %s", uuid) - cloudLogger.Disclose("Disk UUID: " + uuid) + log.Info(fmt.Sprintf("Disk UUID: %s", uuid)) } nodeBootstrapped, err := initialize.IsNodeBootstrapped(openDevice) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to check if node was previously bootstrapped") + log.With(slog.Any("error", err)).Error("Failed to check if node was previously bootstrapped") + reboot.Reboot(fmt.Errorf("checking if node was previously bootstrapped: %w", err)) } if nodeBootstrapped { if err := kube.StartKubelet(); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to restart kubelet") + log.With(slog.Any("error", err)).Error("Failed to restart kubelet") + reboot.Reboot(fmt.Errorf("restarting kubelet: %w", err)) } return } nodeLock := nodelock.New(openDevice) - initServer, err := initserver.New(context.Background(), nodeLock, kube, issuer, fileHandler, metadata, log) + initServer, err := initserver.New(context.Background(), nodeLock, kube, issuer, disk, fileHandler, metadata, log) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create init server") + log.With(slog.Any("error", err)).Error("Failed to create init server") + reboot.Reboot(fmt.Errorf("creating init server: %w", err)) } dialer := dialer.New(issuer, nil, &net.Dialer{}) - joinClient := joinclient.New(nodeLock, dialer, kube, metadata, log) + joinClient := joinclient.New(nodeLock, dialer, kube, metadata, disk, log) cleaner := clean.New().With(initServer).With(joinClient) go cleaner.Start() defer cleaner.Done() - joinClient.Start(cleaner) + var wg sync.WaitGroup - if err := initServer.Serve(bindIP, bindPort, cleaner); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to serve init server") - } + wg.Add(1) + go func() { + defer wg.Done() + if err := joinClient.Start(cleaner); err != nil { + log.With(slog.Any("error", err)).Error("Failed to join cluster") + markDiskForReset(disk) + reboot.Reboot(fmt.Errorf("joining cluster: %w", err)) + } + }() - log.Infof("bootstrapper done") - cloudLogger.Disclose("bootstrapper done") + wg.Add(1) + go func() { + defer wg.Done() + if err := initServer.Serve(bindIP, bindPort, cleaner); err != nil { + log.With(slog.Any("error", err)).Error("Failed to serve init server") + markDiskForReset(disk) + reboot.Reboot(fmt.Errorf("serving init server: %w", err)) + } + }() + wg.Wait() + + log.Info("bootstrapper done") } -func getDiskUUID() (string, error) { - disk := diskencryption.New() +func getDiskUUID(disk *diskencryption.DiskEncryption) (string, error) { free, err := disk.Open() if err != nil { return "", err @@ -90,6 +104,22 @@ func getDiskUUID() (string, error) { return disk.UUID() } +// markDiskForReset sets a token in the cryptsetup header of the disk to indicate the disk should be reset on next boot. +// This is used to reset all state of a node in case the bootstrapper encountered a non recoverable error +// after the node successfully retrieved a join ticket from the JoinService. +// As setting this token is safe as long as we are certain we don't need the data on the disk anymore, we call this +// unconditionally when either the JoinClient or the InitServer encounter an error. +// We don't call it before that, as the node may be restarting after a previous, successful bootstrapping, +// and now encountered a transient error on rejoining the cluster. Wiping the disk now would delete existing data. +func markDiskForReset(disk *diskencryption.DiskEncryption) { + free, err := disk.Open() + if err != nil { + return + } + defer free() + _ = disk.MarkDiskForReset() +} + type clusterInitJoiner interface { joinclient.ClusterJoiner initserver.ClusterInitializer diff --git a/bootstrapper/cmd/bootstrapper/test.go b/bootstrapper/cmd/bootstrapper/test.go index 24bd7d861..0f6707bcd 100644 --- a/bootstrapper/cmd/bootstrapper/test.go +++ b/bootstrapper/cmd/bootstrapper/test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -10,7 +10,6 @@ import ( "context" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions/components" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" @@ -22,13 +21,13 @@ type clusterFake struct{} // InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster. func (c *clusterFake) InitCluster( context.Context, string, string, - bool, components.Components, []string, string, *logger.Logger, + bool, components.Components, []string, string, ) ([]byte, error) { return []byte{}, nil } // JoinCluster will fake joining the current node to an existing cluster. -func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, components.Components, *logger.Logger) error { +func (c *clusterFake) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, components.Components) error { return nil } diff --git a/bootstrapper/initproto/init.pb.go b/bootstrapper/initproto/init.pb.go index 089b06bbe..5ce2213ae 100644 --- a/bootstrapper/initproto/init.pb.go +++ b/bootstrapper/initproto/init.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v4.22.1 +// protoc-gen-go v1.36.6 +// protoc v5.29.1 // source: bootstrapper/initproto/init.proto package initproto @@ -16,6 +16,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -26,10 +27,7 @@ const ( ) type InitRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` KmsUri string `protobuf:"bytes,1,opt,name=kms_uri,json=kmsUri,proto3" json:"kms_uri,omitempty"` StorageUri string `protobuf:"bytes,2,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"` MeasurementSalt []byte `protobuf:"bytes,3,opt,name=measurement_salt,json=measurementSalt,proto3" json:"measurement_salt,omitempty"` @@ -40,15 +38,15 @@ type InitRequest struct { ClusterName string `protobuf:"bytes,9,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"` ApiserverCertSans []string `protobuf:"bytes,10,rep,name=apiserver_cert_sans,json=apiserverCertSans,proto3" json:"apiserver_cert_sans,omitempty"` ServiceCidr string `protobuf:"bytes,11,opt,name=service_cidr,json=serviceCidr,proto3" json:"service_cidr,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InitRequest) Reset() { *x = InitRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_bootstrapper_initproto_init_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_bootstrapper_initproto_init_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InitRequest) String() string { @@ -59,7 +57,7 @@ func (*InitRequest) ProtoMessage() {} func (x *InitRequest) ProtoReflect() protoreflect.Message { mi := &file_bootstrapper_initproto_init_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -145,25 +143,22 @@ func (x *InitRequest) GetServiceCidr() string { } type InitResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Kind: + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Kind: // // *InitResponse_InitSuccess // *InitResponse_InitFailure // *InitResponse_Log - Kind isInitResponse_Kind `protobuf_oneof:"kind"` + Kind isInitResponse_Kind `protobuf_oneof:"kind"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InitResponse) Reset() { *x = InitResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_bootstrapper_initproto_init_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_bootstrapper_initproto_init_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InitResponse) String() string { @@ -174,7 +169,7 @@ func (*InitResponse) ProtoMessage() {} func (x *InitResponse) ProtoReflect() protoreflect.Message { mi := &file_bootstrapper_initproto_init_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -189,30 +184,36 @@ func (*InitResponse) Descriptor() ([]byte, []int) { return file_bootstrapper_initproto_init_proto_rawDescGZIP(), []int{1} } -func (m *InitResponse) GetKind() isInitResponse_Kind { - if m != nil { - return m.Kind +func (x *InitResponse) GetKind() isInitResponse_Kind { + if x != nil { + return x.Kind } return nil } func (x *InitResponse) GetInitSuccess() *InitSuccessResponse { - if x, ok := x.GetKind().(*InitResponse_InitSuccess); ok { - return x.InitSuccess + if x != nil { + if x, ok := x.Kind.(*InitResponse_InitSuccess); ok { + return x.InitSuccess + } } return nil } func (x *InitResponse) GetInitFailure() *InitFailureResponse { - if x, ok := x.GetKind().(*InitResponse_InitFailure); ok { - return x.InitFailure + if x != nil { + if x, ok := x.Kind.(*InitResponse_InitFailure); ok { + return x.InitFailure + } } return nil } func (x *InitResponse) GetLog() *LogResponseType { - if x, ok := x.GetKind().(*InitResponse_Log); ok { - return x.Log + if x != nil { + if x, ok := x.Kind.(*InitResponse_Log); ok { + return x.Log + } } return nil } @@ -240,22 +241,19 @@ func (*InitResponse_InitFailure) isInitResponse_Kind() {} func (*InitResponse_Log) isInitResponse_Kind() {} type InitSuccessResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Kubeconfig []byte `protobuf:"bytes,1,opt,name=kubeconfig,proto3" json:"kubeconfig,omitempty"` + OwnerId []byte `protobuf:"bytes,2,opt,name=owner_id,json=ownerId,proto3" json:"owner_id,omitempty"` + ClusterId []byte `protobuf:"bytes,3,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` unknownFields protoimpl.UnknownFields - - Kubeconfig []byte `protobuf:"bytes,1,opt,name=kubeconfig,proto3" json:"kubeconfig,omitempty"` - OwnerId []byte `protobuf:"bytes,2,opt,name=owner_id,json=ownerId,proto3" json:"owner_id,omitempty"` - ClusterId []byte `protobuf:"bytes,3,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` + sizeCache protoimpl.SizeCache } func (x *InitSuccessResponse) Reset() { *x = InitSuccessResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_bootstrapper_initproto_init_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_bootstrapper_initproto_init_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InitSuccessResponse) String() string { @@ -266,7 +264,7 @@ func (*InitSuccessResponse) ProtoMessage() {} func (x *InitSuccessResponse) ProtoReflect() protoreflect.Message { mi := &file_bootstrapper_initproto_init_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -303,20 +301,17 @@ func (x *InitSuccessResponse) GetClusterId() []byte { } type InitFailureResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` unknownFields protoimpl.UnknownFields - - Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + sizeCache protoimpl.SizeCache } func (x *InitFailureResponse) Reset() { *x = InitFailureResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_bootstrapper_initproto_init_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_bootstrapper_initproto_init_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InitFailureResponse) String() string { @@ -327,7 +322,7 @@ func (*InitFailureResponse) ProtoMessage() {} func (x *InitFailureResponse) ProtoReflect() protoreflect.Message { mi := &file_bootstrapper_initproto_init_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -350,20 +345,17 @@ func (x *InitFailureResponse) GetError() string { } type LogResponseType struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Log []byte `protobuf:"bytes,1,opt,name=log,proto3" json:"log,omitempty"` unknownFields protoimpl.UnknownFields - - Log []byte `protobuf:"bytes,1,opt,name=log,proto3" json:"log,omitempty"` + sizeCache protoimpl.SizeCache } func (x *LogResponseType) Reset() { *x = LogResponseType{} - if protoimpl.UnsafeEnabled { - mi := &file_bootstrapper_initproto_init_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_bootstrapper_initproto_init_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *LogResponseType) String() string { @@ -374,7 +366,7 @@ func (*LogResponseType) ProtoMessage() {} func (x *LogResponseType) ProtoReflect() protoreflect.Message { mi := &file_bootstrapper_initproto_init_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -397,23 +389,20 @@ func (x *LogResponseType) GetLog() []byte { } type KubernetesComponent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"` + InstallPath string `protobuf:"bytes,3,opt,name=install_path,json=installPath,proto3" json:"install_path,omitempty"` + Extract bool `protobuf:"varint,4,opt,name=extract,proto3" json:"extract,omitempty"` unknownFields protoimpl.UnknownFields - - Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` - Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"` - InstallPath string `protobuf:"bytes,3,opt,name=install_path,json=installPath,proto3" json:"install_path,omitempty"` - Extract bool `protobuf:"varint,4,opt,name=extract,proto3" json:"extract,omitempty"` + sizeCache protoimpl.SizeCache } func (x *KubernetesComponent) Reset() { *x = KubernetesComponent{} - if protoimpl.UnsafeEnabled { - mi := &file_bootstrapper_initproto_init_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_bootstrapper_initproto_init_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *KubernetesComponent) String() string { @@ -424,7 +413,7 @@ func (*KubernetesComponent) ProtoMessage() {} func (x *KubernetesComponent) ProtoReflect() protoreflect.Message { mi := &file_bootstrapper_initproto_init_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -469,98 +458,61 @@ func (x *KubernetesComponent) GetExtract() bool { var File_bootstrapper_initproto_init_proto protoreflect.FileDescriptor -var file_bootstrapper_initproto_init_proto_rawDesc = []byte{ - 0x0a, 0x21, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, - 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x1a, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x63, 0x6f, 0x6d, - 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, 0x2f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, - 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd0, 0x03, 0x0a, 0x0b, 0x49, 0x6e, 0x69, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73, 0x5f, - 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55, 0x72, - 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x69, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, - 0x72, 0x69, 0x12, 0x29, 0x0a, 0x10, 0x6d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x5f, 0x73, 0x61, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0f, 0x6d, 0x65, - 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x61, 0x6c, 0x74, 0x12, 0x2d, 0x0a, - 0x12, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, - 0x6e, 0x65, 0x74, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x10, - 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x4a, 0x0a, 0x15, 0x6b, 0x75, 0x62, 0x65, 0x72, - 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x73, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x52, 0x14, 0x6b, - 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, - 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x61, 0x70, 0x69, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x6e, 0x73, 0x18, 0x0a, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x61, 0x70, 0x69, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, - 0x65, 0x72, 0x74, 0x53, 0x61, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x5f, 0x63, 0x69, 0x64, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x69, 0x64, 0x72, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, - 0x52, 0x19, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, - 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x75, 0x72, 0x69, 0x22, 0xc1, 0x01, 0x0a, 0x0c, - 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, - 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, - 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x3e, 0x0a, 0x0c, - 0x69, 0x6e, 0x69, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, - 0x0b, 0x69, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x29, 0x0a, 0x03, - 0x6c, 0x6f, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x69, 0x6e, 0x69, 0x74, - 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x48, 0x00, 0x52, 0x03, 0x6c, 0x6f, 0x67, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, - 0x6f, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, 0x65, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, - 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, - 0x22, 0x2b, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x23, 0x0a, - 0x0f, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x10, 0x0a, 0x03, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6c, - 0x6f, 0x67, 0x22, 0x78, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, - 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x68, - 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, - 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x50, 0x61, - 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x32, 0x36, 0x0a, 0x03, - 0x41, 0x50, 0x49, 0x12, 0x2f, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, - 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, - 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x30, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, - 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x32, 0x2f, - 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, - 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_bootstrapper_initproto_init_proto_rawDesc = "" + + "\n" + + "!bootstrapper/initproto/init.proto\x12\x04init\x1a-internal/versions/components/components.proto\"\xd0\x03\n" + + "\vInitRequest\x12\x17\n" + + "\akms_uri\x18\x01 \x01(\tR\x06kmsUri\x12\x1f\n" + + "\vstorage_uri\x18\x02 \x01(\tR\n" + + "storageUri\x12)\n" + + "\x10measurement_salt\x18\x03 \x01(\fR\x0fmeasurementSalt\x12-\n" + + "\x12kubernetes_version\x18\x05 \x01(\tR\x11kubernetesVersion\x12)\n" + + "\x10conformance_mode\x18\x06 \x01(\bR\x0fconformanceMode\x12J\n" + + "\x15kubernetes_components\x18\a \x03(\v2\x15.components.ComponentR\x14kubernetesComponents\x12\x1f\n" + + "\vinit_secret\x18\b \x01(\fR\n" + + "initSecret\x12!\n" + + "\fcluster_name\x18\t \x01(\tR\vclusterName\x12.\n" + + "\x13apiserver_cert_sans\x18\n" + + " \x03(\tR\x11apiserverCertSans\x12!\n" + + "\fservice_cidr\x18\v \x01(\tR\vserviceCidrJ\x04\b\x04\x10\x05R\x19cloud_service_account_uri\"\xc1\x01\n" + + "\fInitResponse\x12>\n" + + "\finit_success\x18\x01 \x01(\v2\x19.init.InitSuccessResponseH\x00R\vinitSuccess\x12>\n" + + "\finit_failure\x18\x02 \x01(\v2\x19.init.InitFailureResponseH\x00R\vinitFailure\x12)\n" + + "\x03log\x18\x03 \x01(\v2\x15.init.LogResponseTypeH\x00R\x03logB\x06\n" + + "\x04kind\"o\n" + + "\x13InitSuccessResponse\x12\x1e\n" + + "\n" + + "kubeconfig\x18\x01 \x01(\fR\n" + + "kubeconfig\x12\x19\n" + + "\bowner_id\x18\x02 \x01(\fR\aownerId\x12\x1d\n" + + "\n" + + "cluster_id\x18\x03 \x01(\fR\tclusterId\"+\n" + + "\x13InitFailureResponse\x12\x14\n" + + "\x05error\x18\x01 \x01(\tR\x05error\"#\n" + + "\x0fLogResponseType\x12\x10\n" + + "\x03log\x18\x01 \x01(\fR\x03log\"x\n" + + "\x13KubernetesComponent\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x12\n" + + "\x04hash\x18\x02 \x01(\tR\x04hash\x12!\n" + + "\finstall_path\x18\x03 \x01(\tR\vinstallPath\x12\x18\n" + + "\aextract\x18\x04 \x01(\bR\aextract26\n" + + "\x03API\x12/\n" + + "\x04Init\x12\x11.init.InitRequest\x1a\x12.init.InitResponse0\x01B@Z>github.com/edgelesssys/constellation/v2/bootstrapper/initprotob\x06proto3" var ( file_bootstrapper_initproto_init_proto_rawDescOnce sync.Once - file_bootstrapper_initproto_init_proto_rawDescData = file_bootstrapper_initproto_init_proto_rawDesc + file_bootstrapper_initproto_init_proto_rawDescData []byte ) func file_bootstrapper_initproto_init_proto_rawDescGZIP() []byte { file_bootstrapper_initproto_init_proto_rawDescOnce.Do(func() { - file_bootstrapper_initproto_init_proto_rawDescData = protoimpl.X.CompressGZIP(file_bootstrapper_initproto_init_proto_rawDescData) + file_bootstrapper_initproto_init_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_bootstrapper_initproto_init_proto_rawDesc), len(file_bootstrapper_initproto_init_proto_rawDesc))) }) return file_bootstrapper_initproto_init_proto_rawDescData } var file_bootstrapper_initproto_init_proto_msgTypes = make([]protoimpl.MessageInfo, 6) -var file_bootstrapper_initproto_init_proto_goTypes = []interface{}{ +var file_bootstrapper_initproto_init_proto_goTypes = []any{ (*InitRequest)(nil), // 0: init.InitRequest (*InitResponse)(nil), // 1: init.InitResponse (*InitSuccessResponse)(nil), // 2: init.InitSuccessResponse @@ -588,81 +540,7 @@ func file_bootstrapper_initproto_init_proto_init() { if File_bootstrapper_initproto_init_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_bootstrapper_initproto_init_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InitRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_bootstrapper_initproto_init_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InitResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_bootstrapper_initproto_init_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InitSuccessResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_bootstrapper_initproto_init_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InitFailureResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_bootstrapper_initproto_init_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogResponseType); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_bootstrapper_initproto_init_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KubernetesComponent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_bootstrapper_initproto_init_proto_msgTypes[1].OneofWrappers = []interface{}{ + file_bootstrapper_initproto_init_proto_msgTypes[1].OneofWrappers = []any{ (*InitResponse_InitSuccess)(nil), (*InitResponse_InitFailure)(nil), (*InitResponse_Log)(nil), @@ -671,7 +549,7 @@ func file_bootstrapper_initproto_init_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_bootstrapper_initproto_init_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_bootstrapper_initproto_init_proto_rawDesc), len(file_bootstrapper_initproto_init_proto_rawDesc)), NumEnums: 0, NumMessages: 6, NumExtensions: 0, @@ -682,7 +560,6 @@ func file_bootstrapper_initproto_init_proto_init() { MessageInfos: file_bootstrapper_initproto_init_proto_msgTypes, }.Build() File_bootstrapper_initproto_init_proto = out.File - file_bootstrapper_initproto_init_proto_rawDesc = nil file_bootstrapper_initproto_init_proto_goTypes = nil file_bootstrapper_initproto_init_proto_depIdxs = nil } diff --git a/bootstrapper/internal/addresses/BUILD.bazel b/bootstrapper/internal/addresses/BUILD.bazel new file mode 100644 index 000000000..9311c4a61 --- /dev/null +++ b/bootstrapper/internal/addresses/BUILD.bazel @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "interfaces", + srcs = ["interfaces.go"], + importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/interfaces", + visibility = ["//bootstrapper:__subpackages__"], +) + +go_library( + name = "addresses", + srcs = ["addresses.go"], + importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/addresses", + visibility = ["//bootstrapper:__subpackages__"], +) + +go_test( + name = "addresses_test", + srcs = ["addresses_test.go"], + deps = [ + ":addresses", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/bootstrapper/internal/addresses/addresses.go b/bootstrapper/internal/addresses/addresses.go new file mode 100644 index 000000000..d17563b4f --- /dev/null +++ b/bootstrapper/internal/addresses/addresses.go @@ -0,0 +1,45 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package addresses + +import ( + "net" +) + +// GetMachineNetworkAddresses retrieves all network interface addresses. +func GetMachineNetworkAddresses(interfaces []NetInterface) ([]string, error) { + var addresses []string + + for _, i := range interfaces { + addrs, err := i.Addrs() + if err != nil { + return nil, err + } + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + default: + continue + } + if ip.IsLoopback() { + continue + } + addresses = append(addresses, ip.String()) + } + } + + return addresses, nil +} + +// NetInterface represents a network interface used to get network addresses. +type NetInterface interface { + Addrs() ([]net.Addr, error) +} diff --git a/bootstrapper/internal/addresses/addresses_test.go b/bootstrapper/internal/addresses/addresses_test.go new file mode 100644 index 000000000..ceb881ddf --- /dev/null +++ b/bootstrapper/internal/addresses/addresses_test.go @@ -0,0 +1,67 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package addresses_test + +import ( + "errors" + "net" + "testing" + + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/addresses" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetMachineNetworkAddresses(t *testing.T) { + _, someAddr, err := net.ParseCIDR("10.9.0.1/24") + require.NoError(t, err) + + testCases := map[string]struct { + interfaces []addresses.NetInterface + wantErr bool + }{ + "successful": { + interfaces: []addresses.NetInterface{ + &mockNetInterface{ + addrs: []net.Addr{ + someAddr, + }, + }, + }, + }, + "unsuccessful": { + interfaces: []addresses.NetInterface{ + &mockNetInterface{addrs: nil, err: errors.New("someError")}, + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + addrs, err := addresses.GetMachineNetworkAddresses(tc.interfaces) + + if tc.wantErr { + assert.Error(err) + } else { + assert.Equal([]string{"10.9.0.0"}, addrs) + assert.NoError(err) + } + }) + } +} + +type mockNetInterface struct { + addrs []net.Addr + err error +} + +func (m *mockNetInterface) Addrs() ([]net.Addr, error) { + return m.addrs, m.err +} diff --git a/bootstrapper/internal/certificate/certificate.go b/bootstrapper/internal/certificate/certificate.go index e27836bcd..f99ebae02 100644 --- a/bootstrapper/internal/certificate/certificate.go +++ b/bootstrapper/internal/certificate/certificate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package certificate provides functions to create a certificate request and matching private key. diff --git a/bootstrapper/internal/clean/clean.go b/bootstrapper/internal/clean/clean.go index 71d46b4ec..28d9e795d 100644 --- a/bootstrapper/internal/clean/clean.go +++ b/bootstrapper/internal/clean/clean.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package clean provides functionality to stop a list of services gracefully and synchronously. diff --git a/bootstrapper/internal/clean/clean_test.go b/bootstrapper/internal/clean/clean_test.go index 5426de44b..5083265e0 100644 --- a/bootstrapper/internal/clean/clean_test.go +++ b/bootstrapper/internal/clean/clean_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package clean @@ -16,7 +16,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestNew(t *testing.T) { diff --git a/bootstrapper/internal/diskencryption/diskencryption.go b/bootstrapper/internal/diskencryption/diskencryption.go index eaf97e7ab..5cd3f543b 100644 --- a/bootstrapper/internal/diskencryption/diskencryption.go +++ b/bootstrapper/internal/diskencryption/diskencryption.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package diskencryption handles interaction with a node's state disk. @@ -60,6 +60,11 @@ func (c *DiskEncryption) UpdatePassphrase(passphrase string) error { return c.device.SetConstellationStateDiskToken(cryptsetup.SetDiskInitialized) } +// MarkDiskForReset marks the state disk as not initialized so it may be wiped (reset) on reboot. +func (c *DiskEncryption) MarkDiskForReset() error { + return c.device.SetConstellationStateDiskToken(cryptsetup.SetDiskNotInitialized) +} + // getInitialPassphrase retrieves the initial passphrase used on first boot. func (c *DiskEncryption) getInitialPassphrase() (string, error) { passphrase, err := afero.ReadFile(c.fs, initialKeyPath) diff --git a/bootstrapper/internal/diskencryption/diskencryption_test.go b/bootstrapper/internal/diskencryption/diskencryption_test.go index 0c4e9c9f5..7cb5c6483 100644 --- a/bootstrapper/internal/diskencryption/diskencryption_test.go +++ b/bootstrapper/internal/diskencryption/diskencryption_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package diskencryption @@ -18,7 +18,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestUpdatePassphrase(t *testing.T) { diff --git a/bootstrapper/internal/etcdio/BUILD.bazel b/bootstrapper/internal/etcdio/BUILD.bazel new file mode 100644 index 000000000..7f33bd901 --- /dev/null +++ b/bootstrapper/internal/etcdio/BUILD.bazel @@ -0,0 +1,21 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "etcdio", + srcs = [ + "etcdio.go", + "setioprio_cross.go", + "setioprio_linux.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/etcdio", + visibility = ["//bootstrapper:__subpackages__"], + deps = select({ + "@io_bazel_rules_go//go/platform:android": [ + "@org_golang_x_sys//unix", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "@org_golang_x_sys//unix", + ], + "//conditions:default": [], + }), +) diff --git a/bootstrapper/internal/etcdio/etcdio.go b/bootstrapper/internal/etcdio/etcdio.go new file mode 100644 index 000000000..0befb22ee --- /dev/null +++ b/bootstrapper/internal/etcdio/etcdio.go @@ -0,0 +1,154 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +// The etcdio package provides utilities to manage etcd I/O. +package etcdio + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path" + "strconv" + "time" +) + +var ( + // ErrNoEtcdProcess is returned when no etcd process is found on the node. + ErrNoEtcdProcess = errors.New("no etcd process found on node") + // ErrMultipleEtcdProcesses is returned when multiple etcd processes are found on the node. + ErrMultipleEtcdProcesses = errors.New("multiple etcd processes found on node") +) + +const ( + // Tells the syscall that a process' priority is going to be set. + // See https://elixir.bootlin.com/linux/v6.9.1/source/include/uapi/linux/ioprio.h#L54. + ioPrioWhoProcess = 1 + + // See https://elixir.bootlin.com/linux/v6.9.1/source/include/uapi/linux/ioprio.h#L11. + ioPrioClassShift = 13 + ioPrioNrClasses = 8 + ioPrioClassMask = ioPrioNrClasses - 1 + ioPrioPrioMask = (1 << ioPrioClassShift) - 1 + + targetClass = 1 // Realtime IO class for best scheduling prio + targetPrio = 0 // Highest priority within the class +) + +// Client is a client for managing etcd I/O. +type Client struct { + log *slog.Logger +} + +// NewClient creates a new etcd I/O management client. +func NewClient(log *slog.Logger) *Client { + return &Client{log: log} +} + +// PrioritizeIO tries to prioritize the I/O of the etcd process. +// Since it might be possible that the process just started (if this method is called +// right after the kubelet started), it retries to do its work each second +// until it succeeds or the timeout of 10 seconds is reached. +func (c *Client) PrioritizeIO() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + for { + c.log.Info("Prioritizing etcd I/O") + err := c.setIOPriority() + if err == nil { + // Success, return directly + return + } else if errors.Is(err, ErrNoEtcdProcess) { + c.log.Info("No etcd process found, retrying") + } else { + c.log.Warn("Prioritizing etcd I/O failed", "error", err) + return + } + + select { + case <-ticker.C: + case <-timeout.Done(): + c.log.Warn("Timed out waiting for etcd to start") + return + } + } +} + +// setIOPriority tries to find the etcd process on the node and prioritizes its I/O. +func (c *Client) setIOPriority() error { + // find etcd process(es) + pid, err := c.findEtcdProcess() + if err != nil { + return fmt.Errorf("finding etcd process: %w", err) + } + + // Highest realtime priority value for the etcd process, see https://elixir.bootlin.com/linux/v6.9.1/source/include/uapi/linux/ioprio.h + // for the calculation details. + prioVal := ((targetClass & ioPrioClassMask) << ioPrioClassShift) | (targetPrio & ioPrioPrioMask) + + // see https://man7.org/linux/man-pages/man2/ioprio_set.2.html + ret, _, errno := setioprio(ioPrioWhoProcess, uintptr(pid), uintptr(prioVal)) + if ret != 0 { + return fmt.Errorf("setting I/O priority for etcd: %w", errno) + } + + return nil +} + +// findEtcdProcess tries to find the etcd process on the node. +func (c *Client) findEtcdProcess() (int, error) { + procDir, err := os.Open("/proc") + if err != nil { + return 0, fmt.Errorf("opening /proc: %w", err) + } + defer procDir.Close() + + procEntries, err := procDir.Readdirnames(0) + if err != nil { + return 0, fmt.Errorf("reading /proc: %w", err) + } + + // find etcd process(es) + etcdPIDs := []int{} + for _, f := range procEntries { + // exclude non-pid dirs + if f[0] < '0' || f[0] > '9' { + continue + } + + exe, err := os.Readlink(fmt.Sprintf("/proc/%s/exe", f)) + if err != nil { + continue + } + + if path.Base(exe) != "etcd" { + continue + } + + pid, err := strconv.Atoi(f) + if err != nil { + continue + } + + // add the PID to the list of etcd PIDs + etcdPIDs = append(etcdPIDs, pid) + } + + if len(etcdPIDs) == 0 { + return 0, ErrNoEtcdProcess + } + + if len(etcdPIDs) > 1 { + return 0, ErrMultipleEtcdProcesses + } + + return etcdPIDs[0], nil +} diff --git a/bootstrapper/internal/etcdio/setioprio_cross.go b/bootstrapper/internal/etcdio/setioprio_cross.go new file mode 100644 index 000000000..68e5f8e1e --- /dev/null +++ b/bootstrapper/internal/etcdio/setioprio_cross.go @@ -0,0 +1,17 @@ +//go:build !linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package etcdio + +import ( + "syscall" +) + +func setioprio(_, _, _ uintptr) (uintptr, uintptr, syscall.Errno) { + panic("setioprio not implemented on non-Linux platforms") +} diff --git a/bootstrapper/internal/etcdio/setioprio_linux.go b/bootstrapper/internal/etcdio/setioprio_linux.go new file mode 100644 index 000000000..ecd1df82d --- /dev/null +++ b/bootstrapper/internal/etcdio/setioprio_linux.go @@ -0,0 +1,19 @@ +//go:build linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package etcdio + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +func setioprio(ioPrioWhoProcess, pid, prioVal uintptr) (uintptr, uintptr, syscall.Errno) { + return unix.Syscall(unix.SYS_IOPRIO_SET, ioPrioWhoProcess, pid, prioVal) +} diff --git a/bootstrapper/internal/initserver/BUILD.bazel b/bootstrapper/internal/initserver/BUILD.bazel index 385108c57..c25ec5c4a 100644 --- a/bootstrapper/internal/initserver/BUILD.bazel +++ b/bootstrapper/internal/initserver/BUILD.bazel @@ -8,10 +8,11 @@ go_library( visibility = ["//bootstrapper:__subpackages__"], deps = [ "//bootstrapper/initproto", - "//bootstrapper/internal/diskencryption", + "//bootstrapper/internal/addresses", "//bootstrapper/internal/journald", "//internal/atls", "//internal/attestation", + "//internal/constants", "//internal/crypto", "//internal/file", "//internal/grpc/atlscredentials", @@ -22,12 +23,12 @@ go_library( "//internal/nodestate", "//internal/role", "//internal/versions/components", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//keepalive", "@org_golang_google_grpc//status", "@org_golang_x_crypto//bcrypt", - "@org_uber_go_zap//:zap", + "@org_golang_x_crypto//ssh", ], ) @@ -43,6 +44,7 @@ go_test( "//bootstrapper/initproto", "//internal/atls", "//internal/attestation/variant", + "//internal/constants", "//internal/crypto/testvector", "//internal/file", "//internal/kms/setup", @@ -52,8 +54,9 @@ go_test( "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_x_crypto//bcrypt", + "@org_golang_x_crypto//ssh", "@org_uber_go_goleak//:goleak", ], ) diff --git a/bootstrapper/internal/initserver/initserver.go b/bootstrapper/internal/initserver/initserver.go index 1c9d6c40a..5f118d8c0 100644 --- a/bootstrapper/internal/initserver/initserver.go +++ b/bootstrapper/internal/initserver/initserver.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -20,19 +20,23 @@ package initserver import ( "bufio" "context" + "crypto/ed25519" "errors" "fmt" "io" + "log/slog" "net" + "os" "strings" "sync" "time" "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/addresses" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/journald" "github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" @@ -43,8 +47,8 @@ import ( "github.com/edgelesssys/constellation/v2/internal/nodestate" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions/components" - "go.uber.org/zap" "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/ssh" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/keepalive" @@ -65,10 +69,11 @@ type Server struct { shutdownLock sync.RWMutex initSecretHash []byte + initFailure error kmsURI string - log *logger.Logger + log *slog.Logger journaldCollector journaldCollection @@ -76,8 +81,11 @@ type Server struct { } // New creates a new initialization server. -func New(ctx context.Context, lock locker, kube ClusterInitializer, issuer atls.Issuer, fh file.Handler, metadata MetadataAPI, log *logger.Logger) (*Server, error) { - log = log.Named("initServer") +func New( + ctx context.Context, lock locker, kube ClusterInitializer, issuer atls.Issuer, + disk encryptedDisk, fh file.Handler, metadata MetadataAPI, log *slog.Logger, +) (*Server, error) { + log = log.WithGroup("initServer") initSecretHash, err := metadata.InitSecretHash(ctx) if err != nil { @@ -94,7 +102,7 @@ func New(ctx context.Context, lock locker, kube ClusterInitializer, issuer atls. server := &Server{ nodeLock: lock, - disk: diskencryption.New(), + disk: disk, initializer: kube, fileHandler: fh, issuer: issuer, @@ -106,7 +114,7 @@ func New(ctx context.Context, lock locker, kube ClusterInitializer, issuer atls. grpcServer := grpc.NewServer( grpc.Creds(atlscredentials.New(issuer, nil)), grpc.KeepaliveParams(keepalive.ServerParameters{Time: 15 * time.Second}), - log.Named("gRPC").GetServerUnaryInterceptor(), + logger.GetServerUnaryInterceptor(logger.GRPCLogger(log)), ) initproto.RegisterAPIServer(grpcServer, server) @@ -122,51 +130,48 @@ func (s *Server) Serve(ip, port string, cleaner cleaner) error { return fmt.Errorf("failed to listen: %w", err) } - s.log.Infof("Starting") - return s.grpcServer.Serve(lis) + s.log.Info("Starting") + err = s.grpcServer.Serve(lis) + + // If Init failed, we mark the disk for reset, so the node can restart the process + // In this case we don't care about any potential errors from the grpc server + if s.initFailure != nil { + s.log.Error("Fatal error during Init request", "error", s.initFailure) + return err + } + + return err } // Init initializes the cluster. -func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServer) (err error) { +func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServer) (retErr error) { // Acquire lock to prevent shutdown while Init is still running s.shutdownLock.RLock() defer s.shutdownLock.RUnlock() - log := s.log.With(zap.String("peer", grpclog.PeerAddrFromContext(stream.Context()))) - log.Infof("Init called") + log := s.log.With(slog.String("peer", grpclog.PeerAddrFromContext(stream.Context()))) + log.Info("Init called") s.kmsURI = req.KmsUri if err := bcrypt.CompareHashAndPassword(s.initSecretHash, req.InitSecret); err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "invalid init secret %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "invalid init secret %s", err))) } cloudKms, err := kmssetup.KMS(stream.Context(), req.StorageUri, req.KmsUri) if err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "creating kms client: %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "creating kms client: %s", err))) } // generate values for cluster attestation clusterID, err := deriveMeasurementValues(stream.Context(), req.MeasurementSalt, cloudKms) if err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "deriving measurement values: %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "deriving measurement values: %s", err))) } nodeLockAcquired, err := s.nodeLock.TryLockOnce(clusterID) if err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "locking node: %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "locking node: %s", err))) } if !nodeLockAcquired { // The join client seems to already have a connection to an @@ -174,7 +179,7 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe // init does not make sense, so we just stop. // // The server stops itself after the current call is done. - log.Warnf("Node is already in a join process") + log.Warn("Node is already in a join process") err = status.Error(codes.FailedPrecondition, "node is already being activated") @@ -188,12 +193,12 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe // since we are bootstrapping a new one. // Any errors following this call will result in a failed node that may not join any cluster. s.cleaner.Clean() + defer func() { + s.initFailure = retErr + }() if err := s.setupDisk(stream.Context(), cloudKms); err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "setting up disk: %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "setting up disk: %s", err))) } state := nodestate.NodeState{ @@ -201,10 +206,63 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe MeasurementSalt: req.MeasurementSalt, } if err := state.ToFile(s.fileHandler); err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "persisting node state: %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "persisting node state: %s", err))) + } + + // Derive the emergency ssh CA key + key, err := cloudKms.GetDEK(stream.Context(), crypto.DEKPrefix+constants.SSHCAKeySuffix, ed25519.SeedSize) + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "retrieving DEK for key derivation: %s", err))) + } + ca, err := crypto.GenerateEmergencySSHCAKey(key) + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "generating emergency SSH CA key: %s", err))) + } + if err := s.fileHandler.Write(constants.SSHCAKeyPath, ssh.MarshalAuthorizedKey(ca.PublicKey()), file.OptMkdirAll); err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "writing ssh CA pubkey: %s", err))) + } + + interfaces, err := net.Interfaces() + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "getting network interfaces: %s", err))) + } + // Needed since go doesn't implicitly convert slices of structs to slices of interfaces + interfacesForFunc := make([]addresses.NetInterface, len(interfaces)) + for i := range interfaces { + interfacesForFunc[i] = &interfaces[i] + } + + principalList, err := addresses.GetMachineNetworkAddresses(interfacesForFunc) + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "failed to get network addresses: %s", err))) + } + hostname, err := os.Hostname() + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "failed to get hostname: %s", err))) + } + + principalList = append(principalList, hostname) + principalList = append(principalList, req.ApiserverCertSans...) + + hostKeyContent, err := s.fileHandler.Read(constants.SSHHostKeyPath) + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "failed to read host SSH key: %s", err))) + } + + hostPrivateKey, err := ssh.ParsePrivateKey(hostKeyContent) + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "failed to parse host SSH key: %s", err))) + } + + hostKeyPubSSH := hostPrivateKey.PublicKey() + + hostCertificate, err := crypto.GenerateSSHHostCertificate(principalList, hostKeyPubSSH, ca) + if err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "generating SSH host certificate: %s", err))) + } + + if err := s.fileHandler.Write(constants.SSHHostCertificatePath, ssh.MarshalAuthorizedKey(hostCertificate), file.OptMkdirAll); err != nil { + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "writing ssh host certificate: %s", err))) } clusterName := req.ClusterName @@ -219,16 +277,12 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe req.KubernetesComponents, req.ApiserverCertSans, req.ServiceCidr, - s.log, ) if err != nil { - if e := s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "initializing cluster: %s", err)); e != nil { - err = errors.Join(err, e) - } - return err + return errors.Join(err, s.sendLogsWithMessage(stream, status.Errorf(codes.Internal, "initializing cluster: %s", err))) } - log.Infof("Init succeeded") + log.Info("Init succeeded") successMessage := &initproto.InitResponse_InitSuccess{ InitSuccess: &initproto.InitSuccessResponse{ @@ -287,14 +341,14 @@ func (s *Server) sendLogsWithMessage(stream initproto.API_InitServer, message er // Stop stops the initialization server gracefully. func (s *Server) Stop() { - s.log.Infof("Stopping") + s.log.Info("Stopping") // Make sure to only stop the server if no Init calls are running s.shutdownLock.Lock() defer s.shutdownLock.Unlock() s.grpcServer.GracefulStop() - s.log.Infof("Stopped") + s.log.Info("Stopped") } func (s *Server) setupDisk(ctx context.Context, cloudKms kms.CloudKMS) error { @@ -342,7 +396,6 @@ type ClusterInitializer interface { kubernetesComponents components.Components, apiServerCertSANs []string, serviceCIDR string, - log *logger.Logger, ) ([]byte, error) } diff --git a/bootstrapper/internal/initserver/initserver_test.go b/bootstrapper/internal/initserver/initserver_test.go index 65c4485b5..cfecdd018 100644 --- a/bootstrapper/internal/initserver/initserver_test.go +++ b/bootstrapper/internal/initserver/initserver_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package initserver @@ -9,9 +9,12 @@ package initserver import ( "bytes" "context" + "crypto/ed25519" + "encoding/pem" "errors" "io" "net" + "os" "strings" "sync" "testing" @@ -20,6 +23,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/initproto" "github.com/edgelesssys/constellation/v2/internal/atls" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/crypto/testvector" "github.com/edgelesssys/constellation/v2/internal/file" kmssetup "github.com/edgelesssys/constellation/v2/internal/kms/setup" @@ -31,6 +35,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/ssh" "google.golang.org/grpc" ) @@ -38,6 +43,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -65,7 +71,10 @@ func TestNew(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - server, err := New(context.TODO(), newFakeLock(), &stubClusterInitializer{}, atls.NewFakeIssuer(variant.Dummy{}), fh, &tc.metadata, logger.NewTest(t)) + server, err := New( + t.Context(), newFakeLock(), &stubClusterInitializer{}, atls.NewFakeIssuer(variant.Dummy{}), + &stubDisk{}, fh, &tc.metadata, logger.NewTest(t), + ) if tc.wantErr { assert.Error(err) return @@ -96,17 +105,31 @@ func TestInit(t *testing.T) { masterSecret := uri.MasterSecret{Key: []byte("secret"), Salt: []byte("salt")} + _, privkey, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + pemHostKey, err := ssh.MarshalPrivateKey(privkey, "") + require.NoError(t, err) + + fsWithHostKey := afero.NewMemMapFs() + hostKeyFile, err := fsWithHostKey.Create(constants.SSHHostKeyPath) + require.NoError(t, err) + _, err = hostKeyFile.Write(pem.EncodeToMemory(pemHostKey)) + require.NoError(t, err) + require.NoError(t, hostKeyFile.Close()) + readOnlyFSWithHostKey := afero.NewReadOnlyFs(fsWithHostKey) + testCases := map[string]struct { - nodeLock *fakeLock - initializer ClusterInitializer - disk encryptedDisk - fileHandler file.Handler - req *initproto.InitRequest - stream stubStream - logCollector stubJournaldCollector - initSecretHash []byte - wantErr bool - wantShutdown bool + nodeLock *fakeLock + initializer ClusterInitializer + disk encryptedDisk + fileHandler file.Handler + req *initproto.InitRequest + stream stubStream + logCollector stubJournaldCollector + initSecretHash []byte + hostkeyDoesntExist bool + wantErr bool + wantShutdown bool }{ "successful init": { nodeLock: newFakeLock(), @@ -170,7 +193,7 @@ func TestInit(t *testing.T) { nodeLock: newFakeLock(), initializer: &stubClusterInitializer{}, disk: &stubDisk{}, - fileHandler: file.NewHandler(afero.NewReadOnlyFs(afero.NewMemMapFs())), + fileHandler: file.NewHandler(readOnlyFSWithHostKey), req: &initproto.InitRequest{InitSecret: initSecret, KmsUri: masterSecret.EncodeToURI(), StorageUri: uri.NoStoreURI}, stream: stubStream{}, logCollector: stubJournaldCollector{logPipe: &stubReadCloser{reader: bytes.NewReader([]byte{})}}, @@ -201,11 +224,31 @@ func TestInit(t *testing.T) { logCollector: stubJournaldCollector{logPipe: &stubReadCloser{reader: bytes.NewReader([]byte{})}}, wantErr: true, }, + "host key doesn't exist": { + nodeLock: newFakeLock(), + initializer: &stubClusterInitializer{}, + disk: &stubDisk{}, + fileHandler: file.NewHandler(afero.NewMemMapFs()), + initSecretHash: initSecretHash, + req: &initproto.InitRequest{InitSecret: initSecret, KmsUri: masterSecret.EncodeToURI(), StorageUri: uri.NoStoreURI}, + stream: stubStream{}, + logCollector: stubJournaldCollector{logPipe: &stubReadCloser{reader: bytes.NewReader([]byte{})}}, + hostkeyDoesntExist: true, + wantShutdown: true, + wantErr: true, + }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) + require := require.New(t) + + if _, err := tc.fileHandler.Stat(constants.SSHHostKeyPath); errors.Is(err, os.ErrNotExist) { + if !tc.hostkeyDoesntExist { + require.NoError(tc.fileHandler.Write(constants.SSHHostKeyPath, pem.EncodeToMemory(pemHostKey), file.OptMkdirAll)) + } + } serveStopper := newStubServeStopper() server := &Server{ @@ -348,9 +391,9 @@ func TestSetupDisk(t *testing.T) { masterSecret := uri.MasterSecret{Key: tc.masterKey, Salt: tc.salt} - cloudKms, err := kmssetup.KMS(context.Background(), uri.NoStoreURI, masterSecret.EncodeToURI()) + cloudKms, err := kmssetup.KMS(t.Context(), uri.NoStoreURI, masterSecret.EncodeToURI()) require.NoError(err) - assert.NoError(server.setupDisk(context.Background(), cloudKms)) + assert.NoError(server.setupDisk(t.Context(), cloudKms)) }) } } @@ -379,6 +422,10 @@ func (d *fakeDisk) UpdatePassphrase(passphrase string) error { return nil } +func (d *fakeDisk) MarkDiskForReset() error { + return nil +} + type stubDisk struct { openErr error uuid string @@ -400,6 +447,10 @@ func (d *stubDisk) UpdatePassphrase(string) error { return d.updatePassphraseErr } +func (d *stubDisk) MarkDiskForReset() error { + return nil +} + type stubClusterInitializer struct { initClusterKubeconfig []byte initClusterErr error @@ -407,7 +458,7 @@ type stubClusterInitializer struct { func (i *stubClusterInitializer) InitCluster( context.Context, string, string, - bool, components.Components, []string, string, *logger.Logger, + bool, components.Components, []string, string, ) ([]byte, error) { return i.initClusterKubeconfig, i.initClusterErr } diff --git a/bootstrapper/internal/joinclient/BUILD.bazel b/bootstrapper/internal/joinclient/BUILD.bazel index c7d6307b5..a5424ec2d 100644 --- a/bootstrapper/internal/joinclient/BUILD.bazel +++ b/bootstrapper/internal/joinclient/BUILD.bazel @@ -7,13 +7,12 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/joinclient", visibility = ["//bootstrapper:__subpackages__"], deps = [ + "//bootstrapper/internal/addresses", "//bootstrapper/internal/certificate", - "//bootstrapper/internal/diskencryption", "//internal/attestation", "//internal/cloud/metadata", "//internal/constants", "//internal/file", - "//internal/logger", "//internal/nodestate", "//internal/role", "//internal/versions/components", @@ -22,8 +21,8 @@ go_library( "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_kubernetes//cmd/kubeadm/app/constants", "@io_k8s_utils//clock", - "@org_golang_google_grpc//:go_default_library", - "@org_uber_go_zap//:zap", + "@org_golang_google_grpc//:grpc", + "@org_golang_x_crypto//ssh", ], ) @@ -38,6 +37,7 @@ go_test( deps = [ "//internal/cloud/metadata", "//internal/constants", + "//internal/crypto", "//internal/file", "//internal/grpc/atlscredentials", "//internal/grpc/dialer", @@ -51,7 +51,8 @@ go_test( "@com_github_stretchr_testify//require", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_utils//clock/testing", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", + "@org_golang_x_crypto//ssh", "@org_uber_go_goleak//:goleak", ], ) diff --git a/bootstrapper/internal/joinclient/joinclient.go b/bootstrapper/internal/joinclient/joinclient.go index 7f8857419..706efe376 100644 --- a/bootstrapper/internal/joinclient/joinclient.go +++ b/bootstrapper/internal/joinclient/joinclient.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -21,25 +21,25 @@ import ( "context" "errors" "fmt" + "log/slog" "net" + "os" "path/filepath" "strconv" - "sync" "time" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/addresses" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption" "github.com/edgelesssys/constellation/v2/internal/attestation" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/file" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/nodestate" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions/components" "github.com/edgelesssys/constellation/v2/joinservice/joinproto" "github.com/spf13/afero" - "go.uber.org/zap" + "golang.org/x/crypto/ssh" "google.golang.org/grpc" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -70,21 +70,19 @@ type JoinClient struct { dialer grpcDialer joiner ClusterJoiner - cleaner cleaner metadataAPI MetadataAPI - log *logger.Logger + log *slog.Logger - mux sync.Mutex stopC chan struct{} stopDone chan struct{} } // New creates a new JoinClient. -func New(lock locker, dial grpcDialer, joiner ClusterJoiner, meta MetadataAPI, log *logger.Logger) *JoinClient { +func New(lock locker, dial grpcDialer, joiner ClusterJoiner, meta MetadataAPI, disk encryptedDisk, log *slog.Logger) *JoinClient { return &JoinClient{ nodeLock: lock, - disk: diskencryption.New(), + disk: disk, fileHandler: file.NewHandler(afero.NewOsFs()), timeout: timeout, joinTimeout: joinTimeout, @@ -93,99 +91,84 @@ func New(lock locker, dial grpcDialer, joiner ClusterJoiner, meta MetadataAPI, l dialer: dial, joiner: joiner, metadataAPI: meta, - log: log.Named("join-client"), + log: log.WithGroup("join-client"), + + stopC: make(chan struct{}, 1), + stopDone: make(chan struct{}, 1), } } // Start starts the client routine. The client will make the needed API calls to join // the cluster with the role it receives from the metadata API. // After receiving the needed information, the node will join the cluster. -// Multiple calls of start on the same client won't start a second routine if there is -// already a routine running. -func (c *JoinClient) Start(cleaner cleaner) { - c.mux.Lock() - defer c.mux.Unlock() +func (c *JoinClient) Start(cleaner cleaner) error { + c.log.Info("Starting") + ticker := c.clock.NewTicker(c.interval) + defer ticker.Stop() + defer func() { c.stopDone <- struct{}{} }() + defer c.log.Info("Client stopped") - if c.stopC != nil { // daemon already running - return + diskUUID, err := c.getDiskUUID() + if err != nil { + c.log.With(slog.Any("error", err)).Error("Failed to get disk UUID") + return err + } + c.diskUUID = diskUUID + + for { + err := c.getNodeMetadata() + if err == nil { + c.log.With(slog.String("role", c.role.String()), slog.String("name", c.nodeName)).Info("Received own instance metadata") + break + } + c.log.With(slog.Any("error", err)).Error("Failed to retrieve instance metadata") + + c.log.With(slog.Duration("interval", c.interval)).Info("Sleeping") + select { + case <-c.stopC: + return nil + case <-ticker.C(): + } } - c.log.Infof("Starting") - c.stopC = make(chan struct{}, 1) - c.stopDone = make(chan struct{}, 1) - c.cleaner = cleaner + var ticket *joinproto.IssueJoinTicketResponse + var kubeletKey []byte - ticker := c.clock.NewTicker(c.interval) - go func() { - defer ticker.Stop() - defer func() { c.stopDone <- struct{}{} }() - defer c.log.Infof("Client stopped") - - diskUUID, err := c.getDiskUUID() - if err != nil { - c.log.With(zap.Error(err)).Errorf("Failed to get disk UUID") - return + for { + ticket, kubeletKey, err = c.tryJoinWithAvailableServices() + if err == nil { + c.log.Info("Successfully retrieved join ticket, starting Kubernetes node") + break } - c.diskUUID = diskUUID + c.log.With(slog.Any("error", err)).Warn("Join failed for all available endpoints") - for { - err := c.getNodeMetadata() - if err == nil { - c.log.With(zap.String("role", c.role.String()), zap.String("name", c.nodeName)).Infof("Received own instance metadata") - break - } - c.log.With(zap.Error(err)).Errorf("Failed to retrieve instance metadata") - - c.log.With(zap.Duration("interval", c.interval)).Infof("Sleeping") - select { - case <-c.stopC: - return - case <-ticker.C(): - } + c.log.With(slog.Duration("interval", c.interval)).Info("Sleeping") + select { + case <-c.stopC: + return nil + case <-ticker.C(): } + } - for { - err := c.tryJoinWithAvailableServices() - if err == nil { - c.log.Infof("Joined successfully. Client is shutting down") - return - } else if isUnrecoverable(err) { - c.log.With(zap.Error(err)).Errorf("Unrecoverable error occurred") - return - } - c.log.With(zap.Error(err)).Warnf("Join failed for all available endpoints") + if err := c.startNodeAndJoin(ticket, kubeletKey, cleaner); err != nil { + c.log.With(slog.Any("error", err)).Error("Failed to start node and join cluster") + return err + } - c.log.With(zap.Duration("interval", c.interval)).Infof("Sleeping") - select { - case <-c.stopC: - return - case <-ticker.C(): - } - } - }() + return nil } // Stop stops the client and blocks until the client's routine is stopped. func (c *JoinClient) Stop() { - c.mux.Lock() - defer c.mux.Unlock() - - if c.stopC == nil { // daemon not running - return - } - - c.log.Infof("Stopping") + c.log.Info("Stopping") c.stopC <- struct{}{} <-c.stopDone - c.stopC = nil - c.stopDone = nil - - c.log.Infof("Stopped") + c.log.Info("Stopped") } -func (c *JoinClient) tryJoinWithAvailableServices() error { +func (c *JoinClient) tryJoinWithAvailableServices() (ticket *joinproto.IssueJoinTicketResponse, kubeletKey []byte, err error) { ctx, cancel := c.timeoutCtx() defer cancel() @@ -193,75 +176,106 @@ func (c *JoinClient) tryJoinWithAvailableServices() error { endpoint, _, err := c.metadataAPI.GetLoadBalancerEndpoint(ctx) if err != nil { - return fmt.Errorf("failed to get load balancer endpoint: %w", err) + c.log.Warn("Failed to get load balancer endpoint", "err", err) } endpoints = append(endpoints, endpoint) ips, err := c.getControlPlaneIPs(ctx) if err != nil { - return fmt.Errorf("failed to get control plane IPs: %w", err) + c.log.Warn("Failed to get control plane IPs", "err", err) } endpoints = append(endpoints, ips...) if len(endpoints) == 0 { - return errors.New("no control plane IPs found") + return nil, nil, errors.New("no control plane IPs found") } + var joinErrs error for _, endpoint := range endpoints { - err = c.join(net.JoinHostPort(endpoint, strconv.Itoa(constants.JoinServiceNodePort))) + ticket, kubeletKey, err := c.requestJoinTicket(net.JoinHostPort(endpoint, strconv.Itoa(constants.JoinServiceNodePort))) if err == nil { - return nil - } - if isUnrecoverable(err) { - return err + return ticket, kubeletKey, nil } + + joinErrs = errors.Join(joinErrs, err) } - return err + return nil, nil, fmt.Errorf("trying to join on all endpoints %v: %w", endpoints, joinErrs) } -func (c *JoinClient) join(serviceEndpoint string) error { +func (c *JoinClient) requestJoinTicket(serviceEndpoint string) (ticket *joinproto.IssueJoinTicketResponse, kubeletKey []byte, err error) { ctx, cancel := c.timeoutCtx() defer cancel() certificateRequest, kubeletKey, err := certificate.GetKubeletCertificateRequest(c.nodeName, c.validIPs) if err != nil { - return err + return nil, nil, err } - conn, err := c.dialer.Dial(ctx, serviceEndpoint) + interfaces, err := net.Interfaces() if err != nil { - c.log.With(zap.String("endpoint", serviceEndpoint), zap.Error(err)).Errorf("Join service unreachable") - return fmt.Errorf("dialing join service endpoint: %w", err) + c.log.With(slog.Any("error", err)).Error("Failed to get network interfaces") + return nil, nil, err + } + // Needed since go doesn't implicitly convert slices of structs to slices of interfaces + interfacesForFunc := make([]addresses.NetInterface, len(interfaces)) + for i := range interfaces { + interfacesForFunc[i] = &interfaces[i] + } + + principalList, err := addresses.GetMachineNetworkAddresses(interfacesForFunc) + if err != nil { + c.log.With(slog.Any("error", err)).Error("Failed to get network addresses") + return nil, nil, err + } + hostname, err := os.Hostname() + if err != nil { + c.log.With(slog.Any("error", err)).Error("Failed to get hostname") + return nil, nil, err + } + principalList = append(principalList, hostname) + + hostKeyData, err := c.fileHandler.Read(constants.SSHHostKeyPath) + if err != nil { + c.log.With(slog.Any("error", err)).Error("Failed to read SSH host key file") + return nil, nil, err + } + + hostKey, err := ssh.ParsePrivateKey(hostKeyData) + if err != nil { + c.log.With(slog.Any("error", err)).Error("Failed to parse SSH host key file") + return nil, nil, err + } + hostKeyPubSSH := hostKey.PublicKey() + + conn, err := c.dialer.Dial(serviceEndpoint) + if err != nil { + c.log.With(slog.String("endpoint", serviceEndpoint), slog.Any("error", err)).Error("Join service unreachable") + return nil, nil, fmt.Errorf("dialing join service endpoint: %w", err) } defer conn.Close() protoClient := joinproto.NewAPIClient(conn) req := &joinproto.IssueJoinTicketRequest{ - DiskUuid: c.diskUUID, - CertificateRequest: certificateRequest, - IsControlPlane: c.role == role.ControlPlane, + DiskUuid: c.diskUUID, + CertificateRequest: certificateRequest, + IsControlPlane: c.role == role.ControlPlane, + HostPublicKey: hostKeyPubSSH.Marshal(), + HostCertificatePrincipals: principalList, } - ticket, err := protoClient.IssueJoinTicket(ctx, req) + ticket, err = protoClient.IssueJoinTicket(ctx, req) if err != nil { - c.log.With(zap.String("endpoint", serviceEndpoint), zap.Error(err)).Errorf("Issuing join ticket failed") - return fmt.Errorf("issuing join ticket: %w", err) + c.log.With(slog.String("endpoint", serviceEndpoint), slog.Any("error", err)).Error("Issuing join ticket failed") + return nil, nil, fmt.Errorf("issuing join ticket: %w", err) } - return c.startNodeAndJoin(ticket, kubeletKey) + return ticket, kubeletKey, err } -func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse, kubeletKey []byte) (retErr error) { +func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse, kubeletKey []byte, cleaner cleaner) error { ctx, cancel := context.WithTimeout(context.Background(), c.joinTimeout) defer cancel() - // If an error occurs in this func, the client cannot continue. - defer func() { - if retErr != nil { - retErr = unrecoverableError{retErr} - } - }() - clusterID, err := attestation.DeriveClusterID(ticket.MeasurementSecret, ticket.MeasurementSalt) if err != nil { return err @@ -269,17 +283,18 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse, nodeLockAcquired, err := c.nodeLock.TryLockOnce(clusterID) if err != nil { - c.log.With(zap.Error(err)).Errorf("Acquiring node lock failed") + c.log.With(slog.Any("error", err)).Error("Acquiring node lock failed") return fmt.Errorf("acquiring node lock: %w", err) } if !nodeLockAcquired { // There is already a cluster initialization in progress on // this node, so there is no need to also join the cluster, // as the initializing node is automatically part of the cluster. - return errors.New("node is already being initialized") + c.log.Info("Node is already being initialized. Aborting join process.") + return nil } - c.cleaner.Clean() + cleaner.Clean() if err := c.updateDiskPassphrase(string(ticket.StateDiskKey)); err != nil { return fmt.Errorf("updating disk passphrase: %w", err) @@ -297,6 +312,14 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse, return fmt.Errorf("writing kubelet key: %w", err) } + if err := c.fileHandler.Write(constants.SSHCAKeyPath, ticket.AuthorizedCaPublicKey, file.OptMkdirAll); err != nil { + return fmt.Errorf("writing ssh ca key: %w", err) + } + + if err := c.fileHandler.Write(constants.SSHHostCertificatePath, ticket.HostCertificate, file.OptMkdirAll); err != nil { + return fmt.Errorf("writing ssh host certificate: %w", err) + } + state := nodestate.NodeState{ Role: c.role, MeasurementSalt: ticket.MeasurementSalt, @@ -311,7 +334,16 @@ func (c *JoinClient) startNodeAndJoin(ticket *joinproto.IssueJoinTicketResponse, CACertHashes: []string{ticket.DiscoveryTokenCaCertHash}, } - if err := c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesComponents, c.log); err != nil { + // We currently cannot recover from any failure in this function. Joining the k8s cluster + // sometimes fails transiently, and we don't want to brick the node because of that. + for i := range 3 { + err = c.joiner.JoinCluster(ctx, btd, c.role, ticket.KubernetesComponents) + if err == nil { + break + } + c.log.Error("failed to join k8s cluster", "role", c.role, "attempt", i, "error", err) + } + if err != nil { return fmt.Errorf("joining Kubernetes cluster: %w", err) } @@ -322,12 +354,12 @@ func (c *JoinClient) getNodeMetadata() error { ctx, cancel := c.timeoutCtx() defer cancel() - c.log.Debugf("Requesting node metadata from metadata API") + c.log.Debug("Requesting node metadata from metadata API") inst, err := c.metadataAPI.Self(ctx) if err != nil { return err } - c.log.With(zap.Any("instance", inst)).Debugf("Received node metadata") + c.log.With(slog.Any("instance", inst)).Debug("Received node metadata") if inst.Name == "" { return errors.New("got instance metadata with empty name") @@ -371,7 +403,7 @@ func (c *JoinClient) getDiskUUID() (string, error) { func (c *JoinClient) getControlPlaneIPs(ctx context.Context) ([]string, error) { instances, err := c.metadataAPI.List(ctx) if err != nil { - c.log.With(zap.Error(err)).Errorf("Failed to list instances from metadata API") + c.log.With(slog.Any("error", err)).Error("Failed to list instances from metadata API") return nil, fmt.Errorf("listing instances from metadata API: %w", err) } @@ -382,7 +414,7 @@ func (c *JoinClient) getControlPlaneIPs(ctx context.Context) ([]string, error) { } } - c.log.With(zap.Strings("IPs", ips)).Infof("Received control plane endpoints") + c.log.With(slog.Any("IPs", ips)).Info("Received control plane endpoints") return ips, nil } @@ -404,15 +436,8 @@ func (c *JoinClient) timeoutCtx() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), c.timeout) } -type unrecoverableError struct{ error } - -func isUnrecoverable(err error) bool { - _, ok := err.(unrecoverableError) - return ok -} - type grpcDialer interface { - Dial(ctx context.Context, target string) (*grpc.ClientConn, error) + Dial(target string) (*grpc.ClientConn, error) } // ClusterJoiner has the ability to join a new node to an existing cluster. @@ -423,7 +448,6 @@ type ClusterJoiner interface { args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sComponents components.Components, - log *logger.Logger, ) error } diff --git a/bootstrapper/internal/joinclient/joinclient_test.go b/bootstrapper/internal/joinclient/joinclient_test.go index b527b1f23..0f96edaba 100644 --- a/bootstrapper/internal/joinclient/joinclient_test.go +++ b/bootstrapper/internal/joinclient/joinclient_test.go @@ -1,15 +1,18 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package joinclient import ( "context" + "crypto/ed25519" + "encoding/pem" "errors" "net" + "os" "strconv" "sync" "testing" @@ -17,6 +20,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/grpc/atlscredentials" "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" @@ -29,17 +33,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" + "golang.org/x/crypto/ssh" "google.golang.org/grpc" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" testclock "k8s.io/utils/clock/testing" ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestClient(t *testing.T) { - someErr := errors.New("failed") lockedLock := newFakeLock() aqcuiredLock, lockErr := lockedLock.TryLockOnce(nil) require.True(t, aqcuiredLock) @@ -52,25 +56,83 @@ func TestClient(t *testing.T) { {Role: role.ControlPlane, Name: "node-4", VPCIP: "192.0.2.2"}, {Role: role.ControlPlane, Name: "node-5", VPCIP: "192.0.2.3"}, } + caDerivationKey := make([]byte, 256) + respCaKey := &joinproto.IssueJoinTicketResponse{AuthorizedCaPublicKey: caDerivationKey} + + // TODO: fix test since keys are generated with systemd service + makeIssueJoinTicketAnswerWithValidCert := func(t *testing.T, originalAnswer issueJoinTicketAnswer, fh file.Handler) issueJoinTicketAnswer { + require := require.New(t) + + sshKeyBytes, err := fh.Read(constants.SSHHostKeyPath) + require.NoError(err) + sshKey, err := ssh.ParsePrivateKey(sshKeyBytes) + require.NoError(err) + _, randomCAKey, err := ed25519.GenerateKey(nil) + require.NoError(err) + randomCA, err := ssh.NewSignerFromSigner(randomCAKey) + require.NoError(err) + + cert, err := crypto.GenerateSSHHostCertificate([]string{"asdf"}, sshKey.PublicKey(), randomCA) + require.NoError(err) + + certBytes := ssh.MarshalAuthorizedKey(cert) + + if originalAnswer.resp == nil { + originalAnswer.resp = &joinproto.IssueJoinTicketResponse{HostCertificate: certBytes} + } else { + originalAnswer.resp.HostCertificate = certBytes + } + + return originalAnswer + } + + makeIssueJoinTicketAnswerWithInvalidCert := func(t *testing.T, originalAnswer issueJoinTicketAnswer) issueJoinTicketAnswer { + require := require.New(t) + _, randomCAKey, err := ed25519.GenerateKey(nil) + require.NoError(err) + randomCA, err := ssh.NewSignerFromSigner(randomCAKey) + require.NoError(err) + + randomKey, _, err := ed25519.GenerateKey(nil) + require.NoError(err) + randomSSHKey, err := ssh.NewPublicKey(randomKey) + require.NoError(err) + + cert, err := crypto.GenerateSSHHostCertificate([]string{"asdf"}, randomSSHKey, randomCA) + require.NoError(err) + + certBytes := ssh.MarshalAuthorizedKey(cert) + + if originalAnswer.resp == nil { + originalAnswer.resp = &joinproto.IssueJoinTicketResponse{HostCertificate: certBytes} + } else { + originalAnswer.resp.HostCertificate = certBytes + } + + return originalAnswer + } testCases := map[string]struct { - role role.Role - clusterJoiner *stubClusterJoiner - disk encryptedDisk - nodeLock *fakeLock - apiAnswers []any - wantLock bool - wantJoin bool + role role.Role + clusterJoiner *stubClusterJoiner + disk encryptedDisk + nodeLock *fakeLock + apiAnswers []any + wantLock bool + wantJoin bool + wantNumJoins int + wantNotMatchingCert bool + wantCertNotExisting bool }{ "on worker: metadata self: errors occur": { role: role.Worker, apiAnswers: []any{ - selfAnswer{err: someErr}, - selfAnswer{err: someErr}, - selfAnswer{err: someErr}, + selfAnswer{err: assert.AnError}, + selfAnswer{err: assert.AnError}, + selfAnswer{err: assert.AnError}, selfAnswer{instance: workerSelf}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, clusterJoiner: &stubClusterJoiner{}, nodeLock: newFakeLock(), @@ -78,6 +140,23 @@ func TestClient(t *testing.T) { wantJoin: true, wantLock: true, }, + "on worker: SSH host cert not matching": { + role: role.Worker, + apiAnswers: []any{ + selfAnswer{err: assert.AnError}, + selfAnswer{err: assert.AnError}, + selfAnswer{err: assert.AnError}, + selfAnswer{instance: workerSelf}, + listAnswer{instances: peers}, + issueJoinTicketAnswer{resp: respCaKey}, + }, + clusterJoiner: &stubClusterJoiner{}, + nodeLock: newFakeLock(), + disk: &stubDisk{}, + wantJoin: true, + wantLock: true, + wantNotMatchingCert: true, + }, "on worker: metadata self: invalid answer": { role: role.Worker, apiAnswers: []any{ @@ -86,7 +165,7 @@ func TestClient(t *testing.T) { selfAnswer{instance: metadata.InstanceMetadata{Name: "node-1"}}, selfAnswer{instance: workerSelf}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, clusterJoiner: &stubClusterJoiner{}, nodeLock: newFakeLock(), @@ -98,11 +177,11 @@ func TestClient(t *testing.T) { role: role.Worker, apiAnswers: []any{ selfAnswer{instance: workerSelf}, - listAnswer{err: someErr}, - listAnswer{err: someErr}, - listAnswer{err: someErr}, + listAnswer{err: assert.AnError}, + listAnswer{err: assert.AnError}, + listAnswer{err: assert.AnError}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, clusterJoiner: &stubClusterJoiner{}, nodeLock: newFakeLock(), @@ -118,7 +197,7 @@ func TestClient(t *testing.T) { listAnswer{}, listAnswer{}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, clusterJoiner: &stubClusterJoiner{}, nodeLock: newFakeLock(), @@ -131,11 +210,11 @@ func TestClient(t *testing.T) { apiAnswers: []any{ selfAnswer{instance: workerSelf}, listAnswer{instances: peers}, - issueJoinTicketAnswer{err: someErr}, + issueJoinTicketAnswer{err: assert.AnError}, listAnswer{instances: peers}, - issueJoinTicketAnswer{err: someErr}, + issueJoinTicketAnswer{err: assert.AnError}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, clusterJoiner: &stubClusterJoiner{}, nodeLock: newFakeLock(), @@ -148,11 +227,11 @@ func TestClient(t *testing.T) { apiAnswers: []any{ selfAnswer{instance: controlSelf}, listAnswer{instances: peers}, - issueJoinTicketAnswer{err: someErr}, + issueJoinTicketAnswer{err: assert.AnError}, listAnswer{instances: peers}, - issueJoinTicketAnswer{err: someErr}, + issueJoinTicketAnswer{err: assert.AnError}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, clusterJoiner: &stubClusterJoiner{}, nodeLock: newFakeLock(), @@ -165,48 +244,72 @@ func TestClient(t *testing.T) { apiAnswers: []any{ selfAnswer{instance: controlSelf}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, - clusterJoiner: &stubClusterJoiner{joinClusterErr: someErr}, + clusterJoiner: &stubClusterJoiner{numBadCalls: -1, joinClusterErr: assert.AnError}, nodeLock: newFakeLock(), disk: &stubDisk{}, wantJoin: true, wantLock: true, }, + "on control plane: joinCluster fails transiently": { + role: role.ControlPlane, + apiAnswers: []any{ + selfAnswer{instance: controlSelf}, + listAnswer{instances: peers}, + issueJoinTicketAnswer{resp: respCaKey}, + }, + clusterJoiner: &stubClusterJoiner{numBadCalls: 1, joinClusterErr: assert.AnError}, + nodeLock: newFakeLock(), + disk: &stubDisk{}, + wantJoin: true, + wantLock: true, + wantNumJoins: 2, + }, "on control plane: node already locked": { role: role.ControlPlane, apiAnswers: []any{ selfAnswer{instance: controlSelf}, listAnswer{instances: peers}, - issueJoinTicketAnswer{}, + issueJoinTicketAnswer{resp: respCaKey}, }, - clusterJoiner: &stubClusterJoiner{}, - nodeLock: lockedLock, - disk: &stubDisk{}, - wantLock: true, + clusterJoiner: &stubClusterJoiner{}, + nodeLock: lockedLock, + disk: &stubDisk{}, + wantLock: true, + wantCertNotExisting: true, }, "on control plane: disk open fails": { - role: role.ControlPlane, - clusterJoiner: &stubClusterJoiner{}, - nodeLock: newFakeLock(), - disk: &stubDisk{openErr: someErr}, + role: role.ControlPlane, + clusterJoiner: &stubClusterJoiner{}, + nodeLock: newFakeLock(), + disk: &stubDisk{openErr: assert.AnError}, + wantCertNotExisting: true, }, "on control plane: disk uuid fails": { - role: role.ControlPlane, - clusterJoiner: &stubClusterJoiner{}, - nodeLock: newFakeLock(), - disk: &stubDisk{uuidErr: someErr}, + role: role.ControlPlane, + clusterJoiner: &stubClusterJoiner{}, + nodeLock: newFakeLock(), + disk: &stubDisk{uuidErr: assert.AnError}, + wantCertNotExisting: true, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) + require := require.New(t) clock := testclock.NewFakeClock(time.Now()) metadataAPI := newStubMetadataAPI() fileHandler := file.NewHandler(afero.NewMemMapFs()) + _, hostKey, err := ed25519.GenerateKey(nil) + require.NoError(err) + hostKeyPEM, err := ssh.MarshalPrivateKey(hostKey, "hostkey") + require.NoError(err) + require.NoError(fileHandler.Write(constants.SSHHostKeyPath, pem.EncodeToMemory(hostKeyPEM), file.OptMkdirAll)) + netDialer := testdialer.NewBufconnDialer() dialer := dialer.New(nil, nil, netDialer) @@ -221,6 +324,9 @@ func TestClient(t *testing.T) { metadataAPI: metadataAPI, clock: clock, log: logger.NewTest(t), + + stopC: make(chan struct{}, 1), + stopDone: make(chan struct{}, 1), } serverCreds := atlscredentials.New(nil, nil) @@ -232,7 +338,7 @@ func TestClient(t *testing.T) { go joinServer.Serve(listener) defer joinServer.GracefulStop() - client.Start(stubCleaner{}) + go func() { _ = client.Start(stubCleaner{}) }() for _, a := range tc.apiAnswers { switch a := a.(type) { @@ -241,17 +347,50 @@ func TestClient(t *testing.T) { case listAnswer: metadataAPI.listAnswerC <- a case issueJoinTicketAnswer: - joinserviceAPI.issueJoinTicketAnswerC <- a + var answer issueJoinTicketAnswer + if tc.wantNotMatchingCert { + answer = makeIssueJoinTicketAnswerWithInvalidCert(t, a) + } else { + answer = makeIssueJoinTicketAnswerWithValidCert(t, a, fileHandler) + } + joinserviceAPI.issueJoinTicketAnswerC <- answer } clock.Step(time.Second) } client.Stop() - if tc.wantJoin { - assert.True(tc.clusterJoiner.joinClusterCalled) + if !tc.wantCertNotExisting { + hostCertBytes, err := fileHandler.Read(constants.SSHHostCertificatePath) + require.NoError(err) + hostKeyBytes, err := fileHandler.Read(constants.SSHHostKeyPath) + require.NoError(err) + + hostCertKey, _, _, _, err := ssh.ParseAuthorizedKey(hostCertBytes) + require.NoError(err) + hostCert, ok := hostCertKey.(*ssh.Certificate) + require.True(ok) + + hostKey, err := ssh.ParsePrivateKey(hostKeyBytes) + require.NoError(err) + + if !tc.wantNotMatchingCert { + assert.Equal(hostKey.PublicKey().Marshal(), hostCert.Key.Marshal()) + } else { + assert.NotEqual(hostKey.PublicKey().Marshal(), hostCert.Key.Marshal()) + } } else { - assert.False(tc.clusterJoiner.joinClusterCalled) + _, err := fileHandler.Stat(constants.SSHHostCertificatePath) + require.True(errors.Is(err, os.ErrNotExist)) + } + + if tc.wantJoin { + assert.Greater(tc.clusterJoiner.joinClusterCalled, 0) + } else { + assert.Equal(0, tc.clusterJoiner.joinClusterCalled) + } + if tc.wantNumJoins > 0 { + assert.GreaterOrEqual(tc.clusterJoiner.joinClusterCalled, tc.wantNumJoins) } if tc.wantLock { assert.False(client.nodeLock.TryLockOnce(nil)) // lock should be locked @@ -262,78 +401,6 @@ func TestClient(t *testing.T) { } } -func TestClientConcurrentStartStop(t *testing.T) { - netDialer := testdialer.NewBufconnDialer() - dialer := dialer.New(nil, nil, netDialer) - client := &JoinClient{ - nodeLock: newFakeLock(), - timeout: 30 * time.Second, - interval: 30 * time.Second, - dialer: dialer, - disk: &stubDisk{}, - joiner: &stubClusterJoiner{}, - fileHandler: file.NewHandler(afero.NewMemMapFs()), - metadataAPI: &stubRepeaterMetadataAPI{}, - clock: testclock.NewFakeClock(time.Now()), - log: logger.NewTest(t), - } - - wg := sync.WaitGroup{} - - start := func() { - defer wg.Done() - client.Start(stubCleaner{}) - } - - stop := func() { - defer wg.Done() - client.Stop() - } - - wg.Add(10) - go stop() - go start() - go start() - go stop() - go stop() - go start() - go start() - go stop() - go stop() - go start() - wg.Wait() - - client.Stop() -} - -func TestIsUnrecoverable(t *testing.T) { - assert := assert.New(t) - - some := errors.New("failed") - unrec := unrecoverableError{some} - assert.True(isUnrecoverable(unrec)) - assert.False(isUnrecoverable(some)) -} - -type stubRepeaterMetadataAPI struct { - selfInstance metadata.InstanceMetadata - selfErr error - listInstances []metadata.InstanceMetadata - listErr error -} - -func (s *stubRepeaterMetadataAPI) Self(_ context.Context) (metadata.InstanceMetadata, error) { - return s.selfInstance, s.selfErr -} - -func (s *stubRepeaterMetadataAPI) List(_ context.Context) ([]metadata.InstanceMetadata, error) { - return s.listInstances, s.listErr -} - -func (s *stubRepeaterMetadataAPI) GetLoadBalancerEndpoint(_ context.Context) (string, string, error) { - return "", "", nil -} - type stubMetadataAPI struct { selfAnswerC chan selfAnswer listAnswerC chan listAnswer @@ -397,12 +464,17 @@ type issueJoinTicketAnswer struct { } type stubClusterJoiner struct { - joinClusterCalled bool + joinClusterCalled int + numBadCalls int joinClusterErr error } -func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, components.Components, *logger.Logger) error { - j.joinClusterCalled = true +func (j *stubClusterJoiner) JoinCluster(context.Context, *kubeadm.BootstrapTokenDiscovery, role.Role, components.Components) error { + j.joinClusterCalled++ + if j.numBadCalls == 0 { + return nil + } + j.numBadCalls-- return j.joinClusterErr } @@ -427,6 +499,10 @@ func (d *stubDisk) UpdatePassphrase(string) error { return d.updatePassphraseErr } +func (d *stubDisk) MarkDiskForReset() error { + return nil +} + type stubCleaner struct{} func (c stubCleaner) Clean() {} diff --git a/bootstrapper/internal/journald/journald.go b/bootstrapper/internal/journald/journald.go index bf040a1a0..5e06370b2 100644 --- a/bootstrapper/internal/journald/journald.go +++ b/bootstrapper/internal/journald/journald.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/bootstrapper/internal/journald/journald_test.go b/bootstrapper/internal/journald/journald_test.go index 8bf022aec..b617d10ac 100644 --- a/bootstrapper/internal/journald/journald_test.go +++ b/bootstrapper/internal/journald/journald_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package journald diff --git a/bootstrapper/internal/kubernetes/BUILD.bazel b/bootstrapper/internal/kubernetes/BUILD.bazel index c2255148a..935c3fefd 100644 --- a/bootstrapper/internal/kubernetes/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/BUILD.bazel @@ -11,19 +11,18 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes", visibility = ["//bootstrapper:__subpackages__"], deps = [ + "//bootstrapper/internal/etcdio", "//bootstrapper/internal/kubernetes/k8sapi", "//bootstrapper/internal/kubernetes/kubewaiter", "//internal/cloud/cloudprovider", "//internal/cloud/metadata", "//internal/constants", "//internal/kubernetes", - "//internal/logger", "//internal/role", "//internal/versions/components", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", - "@org_uber_go_zap//:zap", ], ) diff --git a/bootstrapper/internal/kubernetes/cloud_provider.go b/bootstrapper/internal/kubernetes/cloud_provider.go index 39023ea82..8b92826c3 100644 --- a/bootstrapper/internal/kubernetes/cloud_provider.go +++ b/bootstrapper/internal/kubernetes/cloud_provider.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel index 85738d500..968a2b011 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel @@ -19,7 +19,6 @@ go_library( "//internal/file", "//internal/installer", "//internal/kubernetes", - "//internal/logger", "//internal/versions/components", "@com_github_coreos_go_systemd_v22//dbus", "@com_github_spf13_afero//:afero", @@ -29,7 +28,7 @@ go_library( "@io_k8s_kubelet//config/v1beta1", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_kubernetes//cmd/kubeadm/app/constants", - "@org_uber_go_zap//:zap", + "@org_golang_x_mod//semver", ], ) diff --git a/bootstrapper/internal/kubernetes/k8sapi/k8sapi.go b/bootstrapper/internal/kubernetes/k8sapi/k8sapi.go index 72565b30a..d5c1c1f35 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/k8sapi.go +++ b/bootstrapper/internal/kubernetes/k8sapi/k8sapi.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package k8sapi is used to interact with the Kubernetes API to create or update required resources. diff --git a/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go index 5f00f3e1e..1cbf88a9b 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sapi/k8sutil.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package k8sapi @@ -13,6 +13,7 @@ import ( "encoding/pem" "errors" "fmt" + "log/slog" "net" "os" "os/exec" @@ -30,9 +31,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/installer" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/afero" - "go.uber.org/zap" ) const ( @@ -49,7 +48,6 @@ type Client interface { AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string, namespace string) error ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error) AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error - EnforceCoreDNSSpread(ctx context.Context) error PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error } @@ -87,9 +85,8 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, kubernetesCompon // InitCluster instruments kubeadm to initialize the K8s cluster. // On success an admin kubeconfig file is returned. func (k *KubernetesUtil) InitCluster( - ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger, + ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *slog.Logger, ) ([]byte, error) { - // TODO(3u13r): audit policy should be user input auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal() if err != nil { return nil, fmt.Errorf("generating default audit policy: %w", err) @@ -108,7 +105,7 @@ func (k *KubernetesUtil) InitCluster( } // preflight - log.Infof("Running kubeadm preflight checks") + log.Info("Running kubeadm preflight checks") cmd := exec.CommandContext(ctx, constants.KubeadmPath, "init", "phase", "preflight", "-v=5", "--config", initConfigFile.Name()) out, err := cmd.CombinedOutput() if err != nil { @@ -120,7 +117,7 @@ func (k *KubernetesUtil) InitCluster( } // create CA certs - log.Infof("Creating Kubernetes control-plane certificates and keys") + log.Info("Creating Kubernetes control-plane certificates and keys") cmd = exec.CommandContext(ctx, constants.KubeadmPath, "init", "phase", "certs", "all", "-v=5", "--config", initConfigFile.Name()) out, err = cmd.CombinedOutput() if err != nil { @@ -132,20 +129,26 @@ func (k *KubernetesUtil) InitCluster( } // create kubelet key and CA signed certificate for the node - log.Infof("Creating signed kubelet certificate") + log.Info("Creating signed kubelet certificate") if err := k.createSignedKubeletCert(nodeName, ips); err != nil { return nil, fmt.Errorf("creating signed kubelete certificate: %w", err) } // Create static pods directory for all nodes (the Kubelets on the worker nodes also expect the path to exist) - log.Infof("Creating static Pod directory /etc/kubernetes/manifests") + // If the node rebooted after the static pod directory was created, + // the existing directory needs to be removed before we can + // try to init the cluster again. + if err := os.RemoveAll("/etc/kubernetes/manifests"); err != nil { + return nil, fmt.Errorf("removing static pods directory: %w", err) + } + log.Info("Creating static Pod directory /etc/kubernetes/manifests") if err := os.MkdirAll("/etc/kubernetes/manifests", os.ModePerm); err != nil { return nil, fmt.Errorf("creating static pods directory: %w", err) } // initialize the cluster - log.Infof("Initializing the cluster using kubeadm init") - skipPhases := "--skip-phases=preflight,certs" + log.Info("Initializing the cluster using kubeadm init") + skipPhases := "--skip-phases=preflight,certs,addon/coredns" if !conformanceMode { skipPhases += ",addon/kube-proxy" } @@ -159,11 +162,11 @@ func (k *KubernetesUtil) InitCluster( } return nil, fmt.Errorf("kubeadm init: %w", err) } - log.With(zap.String("output", string(out))).Infof("kubeadm init succeeded") + log.With(slog.String("output", string(out))).Info("kubeadm init succeeded") userName := clusterName + "-admin" - log.With(zap.String("userName", userName)).Infof("Creating admin kubeconfig file") + log.With(slog.String("userName", userName)).Info("Creating admin kubeconfig file") cmd = exec.CommandContext( ctx, constants.KubeadmPath, "kubeconfig", "user", "--client-name", userName, "--config", initConfigFile.Name(), "--org", user.SystemPrivilegedGroup, @@ -176,13 +179,12 @@ func (k *KubernetesUtil) InitCluster( } return nil, fmt.Errorf("kubeadm kubeconfig user: %w", err) } - log.Infof("kubeadm kubeconfig user succeeded") + log.Info("kubeadm kubeconfig user succeeded") return out, nil } // JoinCluster joins existing Kubernetes cluster using kubeadm join. -func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error { - // TODO(3u13r): audit policy should be user input +func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log *slog.Logger) error { auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal() if err != nil { return fmt.Errorf("generating default audit policy: %w", err) @@ -201,7 +203,13 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log } // Create static pods directory for all nodes (the Kubelets on the worker nodes also expect the path to exist) - log.Infof("Creating static Pod directory /etc/kubernetes/manifests") + // If the node rebooted after the static pod directory was created, for example + // if a failure during an upgrade occurred, the existing directory needs to be + // removed before we can try to join the cluster again. + if err := os.RemoveAll("/etc/kubernetes/manifests"); err != nil { + return fmt.Errorf("removing static pods directory: %w", err) + } + log.Info("Creating static Pod directory /etc/kubernetes/manifests") if err := os.MkdirAll("/etc/kubernetes/manifests", os.ModePerm); err != nil { return fmt.Errorf("creating static pods directory: %w", err) } @@ -216,7 +224,7 @@ func (k *KubernetesUtil) JoinCluster(ctx context.Context, joinConfig []byte, log } return fmt.Errorf("kubeadm join: %w", err) } - log.With(zap.String("output", string(out))).Infof("kubeadm join succeeded") + log.With(slog.String("output", string(out))).Info("kubeadm join succeeded") return nil } diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index 430839a46..ca431441e 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package k8sapi @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/kubernetes" + "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl cloudProvider = "external" } - return KubeadmInitYAML{ + initConfig := KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), @@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl TLSPrivateKeyFile: certificate.KeyFilename, }, } + + if semver.Compare(clusterVersion, "v1.31.0") >= 0 { + initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true} + } + return initConfig } // JoinConfiguration returns a new kubeadm join configuration. diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go index 66a0d3dd3..bf9b68a64 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package k8sapi @@ -18,7 +18,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestInitConfiguration(t *testing.T) { diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy.go b/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy.go index 8d3679d0a..9968c982f 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package resources diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy_test.go b/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy_test.go index 6889cec05..3f9d08914 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/auditpolicy_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package resources diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/resources.go b/bootstrapper/internal/kubernetes/k8sapi/resources/resources.go index d7dc58c49..b7d4bd86f 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/resources.go +++ b/bootstrapper/internal/kubernetes/k8sapi/resources/resources.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package resources contains Kubernetes configs and policies for Constellation. diff --git a/bootstrapper/internal/kubernetes/k8sapi/systemd.go b/bootstrapper/internal/kubernetes/k8sapi/systemd.go index 7ac800f3a..c12484737 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/systemd.go +++ b/bootstrapper/internal/kubernetes/k8sapi/systemd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package k8sapi diff --git a/bootstrapper/internal/kubernetes/k8sutil.go b/bootstrapper/internal/kubernetes/k8sutil.go index 3c7b55718..3752d8087 100644 --- a/bootstrapper/internal/kubernetes/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sutil.go @@ -1,22 +1,22 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes import ( "context" + "log/slog" "net" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/versions/components" ) type clusterUtil interface { InstallComponents(ctx context.Context, kubernetesComponents components.Components) error - InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *logger.Logger) ([]byte, error) - JoinCluster(ctx context.Context, joinConfig []byte, log *logger.Logger) error + InitCluster(ctx context.Context, initConfig []byte, nodeName, clusterName string, ips []net.IP, conformanceMode bool, log *slog.Logger) ([]byte, error) + JoinCluster(ctx context.Context, joinConfig []byte, log *slog.Logger) error StartKubelet() error } diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index ed587d933..4c09ed783 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package kubernetes provides functionality to bootstrap a Kubernetes cluster, or join an exiting one. @@ -10,20 +10,20 @@ package kubernetes import ( "context" "fmt" + "log/slog" "net" "regexp" "strings" "time" + "github.com/edgelesssys/constellation/v2/bootstrapper/internal/etcdio" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/internal/versions/components" - "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" @@ -41,37 +41,48 @@ type kubeAPIWaiter interface { Wait(ctx context.Context, kubernetesClient kubewaiter.KubernetesClient) error } +type etcdIOPrioritizer interface { + PrioritizeIO() +} + // KubeWrapper implements Cluster interface. type KubeWrapper struct { - cloudProvider string - clusterUtil clusterUtil - kubeAPIWaiter kubeAPIWaiter - configProvider configurationProvider - client k8sapi.Client - providerMetadata ProviderMetadata - getIPAddr func() (string, error) + cloudProvider string + clusterUtil clusterUtil + kubeAPIWaiter kubeAPIWaiter + configProvider configurationProvider + client k8sapi.Client + providerMetadata ProviderMetadata + etcdIOPrioritizer etcdIOPrioritizer + getIPAddr func() (string, error) + + log *slog.Logger } // New creates a new KubeWrapper with real values. func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, - providerMetadata ProviderMetadata, kubeAPIWaiter kubeAPIWaiter, + providerMetadata ProviderMetadata, kubeAPIWaiter kubeAPIWaiter, log *slog.Logger, ) *KubeWrapper { + etcdIOPrioritizer := etcdio.NewClient(log) + return &KubeWrapper{ - cloudProvider: cloudProvider, - clusterUtil: clusterUtil, - kubeAPIWaiter: kubeAPIWaiter, - configProvider: configProvider, - client: client, - providerMetadata: providerMetadata, - getIPAddr: getIPAddr, + cloudProvider: cloudProvider, + clusterUtil: clusterUtil, + kubeAPIWaiter: kubeAPIWaiter, + configProvider: configProvider, + client: client, + providerMetadata: providerMetadata, + getIPAddr: getIPAddr, + log: log, + etcdIOPrioritizer: etcdIOPrioritizer, } } // InitCluster initializes a new Kubernetes cluster and applies pod network provider. func (k *KubeWrapper) InitCluster( - ctx context.Context, versionString, clusterName string, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, serviceCIDR string, log *logger.Logger, + ctx context.Context, versionString, clusterName string, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, serviceCIDR string, ) ([]byte, error) { - log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components") + k.log.With(slog.String("version", versionString)).Info("Installing Kubernetes components") if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil { return nil, err } @@ -79,7 +90,7 @@ func (k *KubeWrapper) InitCluster( var validIPs []net.IP // Step 1: retrieve cloud metadata for Kubernetes configuration - log.Infof("Retrieving node metadata") + k.log.Info("Retrieving node metadata") instance, err := k.providerMetadata.Self(ctx) if err != nil { return nil, fmt.Errorf("retrieving own instance metadata: %w", err) @@ -107,15 +118,15 @@ func (k *KubeWrapper) InitCluster( certSANs := []string{nodeIP} certSANs = append(certSANs, apiServerCertSANs...) - log.With( - zap.String("nodeName", nodeName), - zap.String("providerID", instance.ProviderID), - zap.String("nodeIP", nodeIP), - zap.String("controlPlaneHost", controlPlaneHost), - zap.String("controlPlanePort", controlPlanePort), - zap.String("certSANs", strings.Join(certSANs, ",")), - zap.String("podCIDR", subnetworkPodCIDR), - ).Infof("Setting information for node") + k.log.With( + slog.String("nodeName", nodeName), + slog.String("providerID", instance.ProviderID), + slog.String("nodeIP", nodeIP), + slog.String("controlPlaneHost", controlPlaneHost), + slog.String("controlPlanePort", controlPlanePort), + slog.String("certSANs", strings.Join(certSANs, ",")), + slog.String("podCIDR", subnetworkPodCIDR), + ).Info("Setting information for node") // Step 2: configure kubeadm init config ccmSupported := cloudprovider.FromString(k.cloudProvider) == cloudprovider.Azure || @@ -133,12 +144,16 @@ func (k *KubeWrapper) InitCluster( if err != nil { return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err) } - log.Infof("Initializing Kubernetes cluster") - kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, conformanceMode, log) + + k.log.Info("Initializing Kubernetes cluster") + kubeConfig, err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, clusterName, validIPs, conformanceMode, k.log) if err != nil { return nil, fmt.Errorf("kubeadm init: %w", err) } + k.log.Info("Prioritizing etcd I/O") + k.etcdIOPrioritizer.PrioritizeIO() + err = k.client.Initialize(kubeConfig) if err != nil { return nil, fmt.Errorf("initializing kubectl client: %w", err) @@ -150,10 +165,6 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("waiting for Kubernetes API to be available: %w", err) } - if err := k.client.EnforceCoreDNSSpread(ctx); err != nil { - return nil, fmt.Errorf("configuring CoreDNS deployment: %w", err) - } - // Setup the K8s components ConfigMap. k8sComponentsConfigMap, err := k.setupK8sComponentsConfigMap(ctx, kubernetesComponents, versionString) if err != nil { @@ -178,22 +189,23 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err) } - log.Infof("Setting up internal-config ConfigMap") + k.log.Info("Setting up internal-config ConfigMap") if err := k.setupInternalConfigMap(ctx); err != nil { return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) } + return kubeConfig, nil } // JoinCluster joins existing Kubernetes cluster. -func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sComponents components.Components, log *logger.Logger) error { - log.With("k8sComponents", k8sComponents).Infof("Installing provided kubernetes components") +func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTokenDiscovery, peerRole role.Role, k8sComponents components.Components) error { + k.log.With("k8sComponents", k8sComponents).Info("Installing provided kubernetes components") if err := k.clusterUtil.InstallComponents(ctx, k8sComponents); err != nil { return fmt.Errorf("installing kubernetes components: %w", err) } // Step 1: retrieve cloud metadata for Kubernetes configuration - log.Infof("Retrieving node metadata") + k.log.Info("Retrieving node metadata") instance, err := k.providerMetadata.Self(ctx) if err != nil { return fmt.Errorf("retrieving own instance metadata: %w", err) @@ -213,13 +225,13 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo // override join endpoint to go over lb args.APIServerEndpoint = net.JoinHostPort(loadBalancerHost, loadBalancerPort) - log.With( - zap.String("nodeName", nodeName), - zap.String("providerID", providerID), - zap.String("nodeIP", nodeInternalIP), - zap.String("loadBalancerHost", loadBalancerHost), - zap.String("loadBalancerPort", loadBalancerPort), - ).Infof("Setting information for node") + k.log.With( + slog.String("nodeName", nodeName), + slog.String("providerID", providerID), + slog.String("nodeIP", nodeInternalIP), + slog.String("loadBalancerHost", loadBalancerHost), + slog.String("loadBalancerPort", loadBalancerPort), + ).Info("Setting information for node") // Step 2: configure kubeadm join config ccmSupported := cloudprovider.FromString(k.cloudProvider) == cloudprovider.Azure || @@ -238,11 +250,18 @@ func (k *KubeWrapper) JoinCluster(ctx context.Context, args *kubeadm.BootstrapTo if err != nil { return fmt.Errorf("encoding kubeadm join configuration as YAML: %w", err) } - log.With(zap.String("apiServerEndpoint", args.APIServerEndpoint)).Infof("Joining Kubernetes cluster") - if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, log); err != nil { + + k.log.With(slog.String("apiServerEndpoint", args.APIServerEndpoint)).Info("Joining Kubernetes cluster") + if err := k.clusterUtil.JoinCluster(ctx, joinConfigYAML, k.log); err != nil { return fmt.Errorf("joining cluster: %v; %w ", string(joinConfigYAML), err) } + // If on control plane (and thus with etcd), try to prioritize etcd I/O. + if peerRole == role.ControlPlane { + k.log.Info("Prioritizing etcd I/O") + k.etcdIOPrioritizer.PrioritizeIO() + } + return nil } @@ -302,6 +321,8 @@ func (k *KubeWrapper) StartKubelet() error { return fmt.Errorf("starting kubelet: %w", err) } + k.etcdIOPrioritizer.PrioritizeIO() + return nil } diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index 4bc85af5f..02051bd5d 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes @@ -9,6 +9,7 @@ package kubernetes import ( "context" "errors" + "log/slog" "net" "strconv" "testing" @@ -30,7 +31,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestInitCluster(t *testing.T) { @@ -41,17 +42,19 @@ func TestInitCluster(t *testing.T) { aliasIPRange := "192.0.2.0/24" testCases := map[string]struct { - clusterUtil stubClusterUtil - kubectl stubKubectl - kubeAPIWaiter stubKubeAPIWaiter - providerMetadata ProviderMetadata - wantConfig k8sapi.KubeadmInitYAML - wantErr bool - k8sVersion versions.ValidK8sVersion + clusterUtil stubClusterUtil + kubectl stubKubectl + kubeAPIWaiter stubKubeAPIWaiter + providerMetadata ProviderMetadata + wantConfig k8sapi.KubeadmInitYAML + etcdIOPrioritizer stubEtcdIOPrioritizer + wantErr bool + k8sVersion versions.ValidK8sVersion }{ "kubeadm init works with metadata and loadbalancer": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ Name: nodeName, @@ -84,8 +87,9 @@ func TestInitCluster(t *testing.T) { k8sVersion: versions.Default, }, "kubeadm init fails when annotating itself": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ Name: nodeName, @@ -101,8 +105,9 @@ func TestInitCluster(t *testing.T) { k8sVersion: versions.Default, }, "kubeadm init fails when retrieving metadata self": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfErr: assert.AnError, }, @@ -110,7 +115,8 @@ func TestInitCluster(t *testing.T) { k8sVersion: versions.Default, }, "kubeadm init fails when retrieving metadata loadbalancer ip": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ getLoadBalancerEndpointErr: assert.AnError, }, @@ -122,51 +128,58 @@ func TestInitCluster(t *testing.T) { initClusterErr: assert.AnError, kubeconfig: []byte("someKubeconfig"), }, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - wantErr: true, - k8sVersion: versions.Default, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when deploying cilium": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - providerMetadata: &stubProviderMetadata{}, - wantErr: true, - k8sVersion: versions.Default, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting up constellation-services chart": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - wantErr: true, - k8sVersion: versions.Default, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when reading kubeconfig": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - wantErr: true, - k8sVersion: versions.Default, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when setting up verification service": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - wantErr: true, - k8sVersion: versions.Default, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + wantErr: true, + k8sVersion: versions.Default, }, "kubeadm init fails when waiting for kubeAPI server": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{waitErr: assert.AnError}, - providerMetadata: &stubProviderMetadata{}, - k8sVersion: versions.Default, - wantErr: true, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{waitErr: assert.AnError}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + k8sVersion: versions.Default, + wantErr: true, }, "unsupported k8sVersion fails cluster creation": { - clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, - kubeAPIWaiter: stubKubeAPIWaiter{}, - providerMetadata: &stubProviderMetadata{}, - k8sVersion: "1.19", - wantErr: true, + clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")}, + kubeAPIWaiter: stubKubeAPIWaiter{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + k8sVersion: "1.19", + wantErr: true, }, } @@ -176,18 +189,20 @@ func TestInitCluster(t *testing.T) { require := require.New(t) kube := KubeWrapper{ - cloudProvider: "aws", // provide a valid cloud provider for cilium installation - clusterUtil: &tc.clusterUtil, - providerMetadata: tc.providerMetadata, - kubeAPIWaiter: &tc.kubeAPIWaiter, - configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}}, - client: &tc.kubectl, - getIPAddr: func() (string, error) { return privateIP, nil }, + cloudProvider: "aws", // provide a valid cloud provider for cilium installation + clusterUtil: &tc.clusterUtil, + providerMetadata: tc.providerMetadata, + kubeAPIWaiter: &tc.kubeAPIWaiter, + configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}}, + client: &tc.kubectl, + getIPAddr: func() (string, error) { return privateIP, nil }, + etcdIOPrioritizer: &tc.etcdIOPrioritizer, + log: logger.NewTest(t), } _, err := kube.InitCluster( - context.Background(), string(tc.k8sVersion), "kubernetes", - false, nil, nil, "", logger.NewTest(t), + t.Context(), string(tc.k8sVersion), "kubernetes", + false, nil, nil, "", ) if tc.wantErr { @@ -223,15 +238,17 @@ func TestJoinCluster(t *testing.T) { } testCases := map[string]struct { - clusterUtil stubClusterUtil - providerMetadata ProviderMetadata - wantConfig kubeadm.JoinConfiguration - role role.Role - k8sComponents components.Components - wantErr bool + clusterUtil stubClusterUtil + providerMetadata ProviderMetadata + wantConfig kubeadm.JoinConfiguration + role role.Role + k8sComponents components.Components + etcdIOPrioritizer stubEtcdIOPrioritizer + wantErr bool }{ "kubeadm join worker works with metadata and remote Kubernetes Components": { - clusterUtil: stubClusterUtil{}, + clusterUtil: stubClusterUtil{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", @@ -252,7 +269,8 @@ func TestJoinCluster(t *testing.T) { }, }, "kubeadm join worker works with metadata and local Kubernetes components": { - clusterUtil: stubClusterUtil{}, + clusterUtil: stubClusterUtil{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", @@ -272,7 +290,8 @@ func TestJoinCluster(t *testing.T) { }, }, "kubeadm join worker works with metadata and cloud controller manager": { - clusterUtil: stubClusterUtil{}, + clusterUtil: stubClusterUtil{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", @@ -292,7 +311,8 @@ func TestJoinCluster(t *testing.T) { }, }, "kubeadm join control-plane node works with metadata": { - clusterUtil: stubClusterUtil{}, + clusterUtil: stubClusterUtil{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", @@ -319,7 +339,8 @@ func TestJoinCluster(t *testing.T) { }, }, "kubeadm join worker fails when installing remote Kubernetes components": { - clusterUtil: stubClusterUtil{installComponentsErr: errors.New("error")}, + clusterUtil: stubClusterUtil{installComponentsErr: errors.New("error")}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfResp: metadata.InstanceMetadata{ ProviderID: "provider-id", @@ -332,7 +353,8 @@ func TestJoinCluster(t *testing.T) { wantErr: true, }, "kubeadm join worker fails when retrieving self metadata": { - clusterUtil: stubClusterUtil{}, + clusterUtil: stubClusterUtil{}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, providerMetadata: &stubProviderMetadata{ selfErr: assert.AnError, }, @@ -340,10 +362,11 @@ func TestJoinCluster(t *testing.T) { wantErr: true, }, "kubeadm join worker fails when applying the join config": { - clusterUtil: stubClusterUtil{joinClusterErr: assert.AnError}, - providerMetadata: &stubProviderMetadata{}, - role: role.Worker, - wantErr: true, + clusterUtil: stubClusterUtil{joinClusterErr: assert.AnError}, + etcdIOPrioritizer: stubEtcdIOPrioritizer{}, + providerMetadata: &stubProviderMetadata{}, + role: role.Worker, + wantErr: true, }, } @@ -353,13 +376,15 @@ func TestJoinCluster(t *testing.T) { require := require.New(t) kube := KubeWrapper{ - clusterUtil: &tc.clusterUtil, - providerMetadata: tc.providerMetadata, - configProvider: &stubConfigProvider{}, - getIPAddr: func() (string, error) { return privateIP, nil }, + clusterUtil: &tc.clusterUtil, + providerMetadata: tc.providerMetadata, + configProvider: &stubConfigProvider{}, + getIPAddr: func() (string, error) { return privateIP, nil }, + etcdIOPrioritizer: &tc.etcdIOPrioritizer, + log: logger.NewTest(t), } - err := kube.JoinCluster(context.Background(), joinCommand, tc.role, tc.k8sComponents, logger.NewTest(t)) + err := kube.JoinCluster(t.Context(), joinCommand, tc.role, tc.k8sComponents) if tc.wantErr { assert.Error(err) return @@ -440,7 +465,7 @@ func (s *stubClusterUtil) InstallComponents(_ context.Context, _ components.Comp return s.installComponentsErr } -func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _ bool, _ *logger.Logger) ([]byte, error) { +func (s *stubClusterUtil) InitCluster(_ context.Context, initConfig []byte, _, _ string, _ []net.IP, _ bool, _ *slog.Logger) ([]byte, error) { s.initConfigs = append(s.initConfigs, initConfig) return s.kubeconfig, s.initClusterErr } @@ -465,7 +490,7 @@ func (s *stubClusterUtil) SetupNodeOperator(_ context.Context, _ k8sapi.Client, return s.setupNodeOperatorErr } -func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ *logger.Logger) error { +func (s *stubClusterUtil) JoinCluster(_ context.Context, joinConfig []byte, _ *slog.Logger) error { s.joinConfigs = append(s.joinConfigs, joinConfig) return s.joinClusterErr } @@ -544,3 +569,7 @@ type stubKubeAPIWaiter struct { func (s *stubKubeAPIWaiter) Wait(_ context.Context, _ kubewaiter.KubernetesClient) error { return s.waitErr } + +type stubEtcdIOPrioritizer struct{} + +func (s *stubEtcdIOPrioritizer) PrioritizeIO() {} diff --git a/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter.go b/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter.go index 00e097053..956f8dd0d 100644 --- a/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter.go +++ b/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package kubewaiter is used to wait for the Kubernetes API to be available. @@ -29,7 +29,7 @@ type CloudKubeAPIWaiter struct{} // Wait waits for the Kubernetes API to be available. // Note that the kubernetesClient must have the kubeconfig already set. func (w *CloudKubeAPIWaiter) Wait(ctx context.Context, kubernetesClient KubernetesClient) error { - funcAlwaysRetriable := func(err error) bool { return true } + funcAlwaysRetriable := func(_ error) bool { return true } doer := &kubeDoer{kubeClient: kubernetesClient} retrier := retry.NewIntervalRetrier(doer, 5*time.Second, funcAlwaysRetriable) diff --git a/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter_test.go b/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter_test.go index bb0174ae9..c284bf3aa 100644 --- a/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter_test.go +++ b/bootstrapper/internal/kubernetes/kubewaiter/kubewaiter_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubewaiter @@ -17,7 +17,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestCloudKubeAPIWaiter(t *testing.T) { @@ -39,7 +39,7 @@ func TestCloudKubeAPIWaiter(t *testing.T) { require := require.New(t) waiter := &CloudKubeAPIWaiter{} - ctx, cancel := context.WithTimeout(context.Background(), 0) + ctx, cancel := context.WithTimeout(t.Context(), 0) defer cancel() err := waiter.Wait(ctx, tc.kubeClient) if tc.wantErr { diff --git a/bootstrapper/internal/logging/logger.go b/bootstrapper/internal/logging/logger.go index b14f5668f..a24a5a24f 100644 --- a/bootstrapper/internal/logging/logger.go +++ b/bootstrapper/internal/logging/logger.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package logging provides an interface for logging information to a non-confidential destination diff --git a/bootstrapper/internal/nodelock/nodelock.go b/bootstrapper/internal/nodelock/nodelock.go index 2a3865c8d..973877493 100644 --- a/bootstrapper/internal/nodelock/nodelock.go +++ b/bootstrapper/internal/nodelock/nodelock.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package nodelock handles locking operations on the node. diff --git a/bootstrapper/internal/nodelock/nodelock_test.go b/bootstrapper/internal/nodelock/nodelock_test.go index c5738fec1..967432d6a 100644 --- a/bootstrapper/internal/nodelock/nodelock_test.go +++ b/bootstrapper/internal/nodelock/nodelock_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package nodelock diff --git a/bootstrapper/internal/reboot/BUILD.bazel b/bootstrapper/internal/reboot/BUILD.bazel new file mode 100644 index 000000000..ce71293b3 --- /dev/null +++ b/bootstrapper/internal/reboot/BUILD.bazel @@ -0,0 +1,11 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "reboot", + srcs = [ + "reboot_cross.go", + "reboot_linux.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/bootstrapper/internal/reboot", + visibility = ["//bootstrapper:__subpackages__"], +) diff --git a/bootstrapper/internal/reboot/reboot_cross.go b/bootstrapper/internal/reboot/reboot_cross.go new file mode 100644 index 000000000..48c638c40 --- /dev/null +++ b/bootstrapper/internal/reboot/reboot_cross.go @@ -0,0 +1,14 @@ +//go:build !linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package reboot + +// Reboot is not implemented on non-Linux platforms. +func Reboot(_ error) { + panic("reboot not implemented on non-Linux platforms") +} diff --git a/bootstrapper/internal/reboot/reboot_linux.go b/bootstrapper/internal/reboot/reboot_linux.go new file mode 100644 index 000000000..7b2b8847e --- /dev/null +++ b/bootstrapper/internal/reboot/reboot_linux.go @@ -0,0 +1,29 @@ +//go:build linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package reboot + +import ( + "log/syslog" + "syscall" + "time" +) + +// Reboot writes an error message to the system log and reboots the system. +// We call this instead of os.Exit() since failures in the bootstrapper usually require a node reset. +func Reboot(e error) { + syslogWriter, err := syslog.New(syslog.LOG_EMERG|syslog.LOG_KERN, "bootstrapper") + if err != nil { + _ = syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART) + } + _ = syslogWriter.Err(e.Error()) + _ = syslogWriter.Emerg("bootstrapper has encountered a non recoverable error. Rebooting...") + time.Sleep(time.Minute) // sleep to allow the message to be written to syslog and seen by the user + + _ = syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART) +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 6baaf3f1f..1826812fa 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -61,6 +61,7 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(cmd.NewIAMCmd()) rootCmd.AddCommand(cmd.NewVersionCmd()) rootCmd.AddCommand(cmd.NewInitCmd()) + rootCmd.AddCommand(cmd.NewSSHCmd()) rootCmd.AddCommand(cmd.NewMaaPatchCmd()) return rootCmd diff --git a/cli/internal/cloudcmd/BUILD.bazel b/cli/internal/cloudcmd/BUILD.bazel index 83b394338..946322495 100644 --- a/cli/internal/cloudcmd/BUILD.bazel +++ b/cli/internal/cloudcmd/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "//internal/cloud/cloudprovider", "//internal/cloud/gcpshared", "//internal/cloud/openstack", + "//internal/cloud/openstack/clouds", "//internal/config", "//internal/constants", "//internal/constellation", diff --git a/cli/internal/cloudcmd/apply.go b/cli/internal/cloudcmd/apply.go index 2f419d41a..c3b9210c1 100644 --- a/cli/internal/cloudcmd/apply.go +++ b/cli/internal/cloudcmd/apply.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -15,6 +15,7 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/libvirt" "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -89,7 +90,9 @@ func (a *Applier) Plan(ctx context.Context, conf *config.Config) (bool, error) { } // Apply applies the prepared configuration by creating or updating cloud resources. -func (a *Applier) Apply(ctx context.Context, csp cloudprovider.Provider, withRollback RollbackBehavior) (infra state.Infrastructure, retErr error) { +func (a *Applier) Apply( + ctx context.Context, csp cloudprovider.Provider, attestation variant.Variant, withRollback RollbackBehavior, +) (infra state.Infrastructure, retErr error) { if withRollback { var rollbacker rollbacker switch csp { @@ -105,7 +108,7 @@ func (a *Applier) Apply(ctx context.Context, csp cloudprovider.Provider, withRol if err != nil { return infraState, fmt.Errorf("terraform apply: %w", err) } - if csp == cloudprovider.Azure && infraState.Azure != nil { + if csp == cloudprovider.Azure && attestation.Equal(variant.AzureSEVSNP{}) && infraState.Azure != nil { if err := a.policyPatcher.Patch(ctx, infraState.Azure.AttestationURL); err != nil { return infraState, fmt.Errorf("patching policies: %w", err) } diff --git a/cli/internal/cloudcmd/apply_test.go b/cli/internal/cloudcmd/apply_test.go index ae8fa2ce3..f64b6afb9 100644 --- a/cli/internal/cloudcmd/apply_test.go +++ b/cli/internal/cloudcmd/apply_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -26,7 +27,6 @@ import ( ) func TestApplier(t *testing.T) { - t.Setenv("CONSTELLATION_OPENSTACK_DEV", "1") failOnNonAMD64 := (runtime.GOARCH != "amd64") || (runtime.GOOS != "linux") ip := "192.0.2.1" configWithProvider := func(provider cloudprovider.Provider) *config.Config { @@ -185,14 +185,14 @@ func TestApplier(t *testing.T) { out: &bytes.Buffer{}, } - diff, err := applier.Plan(context.Background(), tc.config) + diff, err := applier.Plan(t.Context(), tc.config) if err != nil { assert.True(tc.wantErr, "unexpected error: %s", err) return } assert.False(diff) - idFile, err := applier.Apply(context.Background(), tc.provider, true) + idFile, err := applier.Apply(t.Context(), tc.provider, tc.config.GetAttestationConfig().GetVariant(), true) if tc.wantErr { assert.Error(err) @@ -303,7 +303,7 @@ func TestPlan(t *testing.T) { cfg := config.Default() cfg.RemoveProviderAndAttestationExcept(cloudprovider.Azure) - diff, err := u.Plan(context.Background(), cfg) + diff, err := u.Plan(t.Context(), cfg) if tc.wantErr { require.Error(err) } else { @@ -352,7 +352,7 @@ func TestApply(t *testing.T) { out: io.Discard, } - _, err := u.Apply(context.Background(), cloudprovider.QEMU, WithoutRollbackOnError) + _, err := u.Apply(t.Context(), cloudprovider.QEMU, variant.QEMUVTPM{}, WithoutRollbackOnError) if tc.wantErr { assert.Error(err) } else { diff --git a/cli/internal/cloudcmd/clients.go b/cli/internal/cloudcmd/clients.go index 75858f19c..897610f80 100644 --- a/cli/internal/cloudcmd/clients.go +++ b/cli/internal/cloudcmd/clients.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd diff --git a/cli/internal/cloudcmd/clients_test.go b/cli/internal/cloudcmd/clients_test.go index ec229156c..fa6985ab1 100644 --- a/cli/internal/cloudcmd/clients_test.go +++ b/cli/internal/cloudcmd/clients_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -23,6 +23,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } diff --git a/cli/internal/cloudcmd/cloudcmd.go b/cli/internal/cloudcmd/cloudcmd.go index 12c6e2879..17ce6bc93 100644 --- a/cli/internal/cloudcmd/cloudcmd.go +++ b/cli/internal/cloudcmd/cloudcmd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/cli/internal/cloudcmd/iam.go b/cli/internal/cloudcmd/iam.go index e5902c842..c02116775 100644 --- a/cli/internal/cloudcmd/iam.go +++ b/cli/internal/cloudcmd/iam.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -91,10 +91,12 @@ type GCPIAMConfig struct { Zone string ProjectID string ServiceAccountID string + NamePrefix string } // AzureIAMConfig holds the necessary values for Azure IAM configuration. type AzureIAMConfig struct { + SubscriptionID string Location string ServicePrincipal string ResourceGroup string @@ -140,6 +142,7 @@ func (c *IAMCreator) createGCP(ctx context.Context, cl tfIAMClient, opts *IAMCon vars := terraform.GCPIAMVariables{ ServiceAccountID: opts.GCP.ServiceAccountID, + NamePrefix: opts.GCP.NamePrefix, Project: opts.GCP.ProjectID, Region: opts.GCP.Region, Zone: opts.GCP.Zone, @@ -157,7 +160,8 @@ func (c *IAMCreator) createGCP(ctx context.Context, cl tfIAMClient, opts *IAMCon return IAMOutput{ CloudProvider: cloudprovider.GCP, GCPOutput: GCPIAMOutput{ - ServiceAccountKey: iamOutput.GCP.SaKey, + ServiceAccountKey: iamOutput.GCP.SaKey, + IAMServiceAccountVM: iamOutput.GCP.ServiceAccountVMMailAddress, }, }, nil } @@ -167,6 +171,7 @@ func (c *IAMCreator) createAzure(ctx context.Context, cl tfIAMClient, opts *IAMC defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel) vars := terraform.AzureIAMVariables{ + SubscriptionID: opts.Azure.SubscriptionID, Location: opts.Azure.Location, ResourceGroup: opts.Azure.ResourceGroup, ServicePrincipal: opts.Azure.ServicePrincipal, @@ -230,7 +235,8 @@ type IAMOutput struct { // GCPIAMOutput contains the output information of a GCP IAM configuration. type GCPIAMOutput struct { - ServiceAccountKey string `json:"serviceAccountID,omitempty"` + ServiceAccountKey string `json:"serviceAccountID,omitempty"` + IAMServiceAccountVM string `json:"iamServiceAccountVM,omitempty"` } // AzureIAMOutput contains the output information of a Microsoft Azure IAM configuration. diff --git a/cli/internal/cloudcmd/iam_test.go b/cli/internal/cloudcmd/iam_test.go index ff198c51c..be865bbd2 100644 --- a/cli/internal/cloudcmd/iam_test.go +++ b/cli/internal/cloudcmd/iam_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -128,7 +128,7 @@ func TestIAMCreator(t *testing.T) { }, } - idFile, err := creator.Create(context.Background(), tc.provider, tc.config) + idFile, err := creator.Create(t.Context(), tc.provider, tc.config) if tc.wantErr { assert.Error(err) @@ -184,7 +184,7 @@ func TestDestroyIAMConfiguration(t *testing.T) { return tc.tfClient, nil }} - err := destroyer.DestroyIAMConfiguration(context.Background(), "", terraform.LogLevelNone) + err := destroyer.DestroyIAMConfiguration(t.Context(), "", terraform.LogLevelNone) if tc.wantErr { assert.Error(err) @@ -278,7 +278,7 @@ func TestGetTfstateServiceAccountKey(t *testing.T) { return tc.cl, nil }} - saKey, err := destroyer.GetTfStateServiceAccountKey(context.Background(), "") + saKey, err := destroyer.GetTfStateServiceAccountKey(t.Context(), "") if tc.wantErr { assert.Error(err) diff --git a/cli/internal/cloudcmd/iamupgrade.go b/cli/internal/cloudcmd/iamupgrade.go index 729af5d29..366b771ca 100644 --- a/cli/internal/cloudcmd/iamupgrade.go +++ b/cli/internal/cloudcmd/iamupgrade.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd diff --git a/cli/internal/cloudcmd/rollback.go b/cli/internal/cloudcmd/rollback.go index 7d894cd2f..bdf14e42c 100644 --- a/cli/internal/cloudcmd/rollback.go +++ b/cli/internal/cloudcmd/rollback.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd diff --git a/cli/internal/cloudcmd/rollback_test.go b/cli/internal/cloudcmd/rollback_test.go index 320dd1745..85ad8d3f7 100644 --- a/cli/internal/cloudcmd/rollback_test.go +++ b/cli/internal/cloudcmd/rollback_test.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd import ( "bytes" - "context" "errors" "testing" @@ -46,7 +45,7 @@ func TestRollbackTerraform(t *testing.T) { } destroyClusterErrOutput := &bytes.Buffer{} - err := rollbacker.rollback(context.Background(), destroyClusterErrOutput, terraform.LogLevelNone) + err := rollbacker.rollback(t.Context(), destroyClusterErrOutput, terraform.LogLevelNone) if tc.wantCleanupErr { assert.Error(err) if tc.tfClient.cleanUpWorkspaceErr == nil { @@ -107,7 +106,7 @@ func TestRollbackQEMU(t *testing.T) { destroyClusterErrOutput := &bytes.Buffer{} - err := rollbacker.rollback(context.Background(), destroyClusterErrOutput, terraform.LogLevelNone) + err := rollbacker.rollback(t.Context(), destroyClusterErrOutput, terraform.LogLevelNone) if tc.wantErr { assert.Error(err) if tc.tfClient.cleanUpWorkspaceErr == nil { diff --git a/cli/internal/cloudcmd/serviceaccount.go b/cli/internal/cloudcmd/serviceaccount.go index 994aaa5b0..81a15dbe3 100644 --- a/cli/internal/cloudcmd/serviceaccount.go +++ b/cli/internal/cloudcmd/serviceaccount.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -13,6 +13,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" + "github.com/edgelesssys/constellation/v2/internal/cloud/openstack/clouds" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constellation" "github.com/edgelesssys/constellation/v2/internal/file" @@ -38,15 +39,23 @@ func GetMarshaledServiceAccountURI(config *config.Config, fileHandler file.Handl } case cloudprovider.OpenStack: + cloudsYAML, err := clouds.ReadCloudsYAML(fileHandler, config.Provider.OpenStack.CloudsYAMLPath) + if err != nil { + return "", fmt.Errorf("reading clouds.yaml: %w", err) + } + cloud, ok := cloudsYAML.Clouds[config.Provider.OpenStack.Cloud] + if !ok { + return "", fmt.Errorf("cloud %q not found in clouds.yaml", config.Provider.OpenStack.Cloud) + } payload.OpenStack = openstack.AccountKey{ - AuthURL: config.Provider.OpenStack.AuthURL, - Username: config.Provider.OpenStack.Username, - Password: config.Provider.OpenStack.Password, - ProjectID: config.Provider.OpenStack.ProjectID, - ProjectName: config.Provider.OpenStack.ProjectName, - UserDomainName: config.Provider.OpenStack.UserDomainName, - ProjectDomainName: config.Provider.OpenStack.ProjectDomainName, - RegionName: config.Provider.OpenStack.RegionName, + AuthURL: cloud.AuthInfo.AuthURL, + Username: cloud.AuthInfo.Username, + Password: cloud.AuthInfo.Password, + ProjectID: cloud.AuthInfo.ProjectID, + ProjectName: cloud.AuthInfo.ProjectName, + UserDomainName: cloud.AuthInfo.UserDomainName, + ProjectDomainName: cloud.AuthInfo.ProjectDomainName, + RegionName: cloud.RegionName, } } diff --git a/cli/internal/cloudcmd/terminate.go b/cli/internal/cloudcmd/terminate.go index 4005afa9a..3cb9cccaa 100644 --- a/cli/internal/cloudcmd/terminate.go +++ b/cli/internal/cloudcmd/terminate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd diff --git a/cli/internal/cloudcmd/terminate_test.go b/cli/internal/cloudcmd/terminate_test.go index 1d9f0232c..1b72b3458 100644 --- a/cli/internal/cloudcmd/terminate_test.go +++ b/cli/internal/cloudcmd/terminate_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -63,7 +63,7 @@ func TestTerminator(t *testing.T) { }, } - err := terminator.Terminate(context.Background(), "", terraform.LogLevelNone) + err := terminator.Terminate(t.Context(), "", terraform.LogLevelNone) if tc.wantErr { assert.Error(err) diff --git a/cli/internal/cloudcmd/tfplan.go b/cli/internal/cloudcmd/tfplan.go index ddcccc72d..7fed5de2a 100644 --- a/cli/internal/cloudcmd/tfplan.go +++ b/cli/internal/cloudcmd/tfplan.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd diff --git a/cli/internal/cloudcmd/tfplan_test.go b/cli/internal/cloudcmd/tfplan_test.go index 5d7ad6a42..f4a2b4f5d 100644 --- a/cli/internal/cloudcmd/tfplan_test.go +++ b/cli/internal/cloudcmd/tfplan_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -54,7 +54,7 @@ func TestTFPlan(t *testing.T) { wantBackup: true, }, "workspace is empty": { - prepareFs: func(require *require.Assertions) file.Handler { + prepareFs: func(_ *require.Assertions) file.Handler { return file.NewHandler(afero.NewMemMapFs()) }, tf: &stubUpgradePlanner{}, @@ -101,7 +101,7 @@ func TestTFPlan(t *testing.T) { fs := tc.prepareFs(require.New(t)) hasDiff, planErr := plan( - context.Background(), tc.tf, fs, io.Discard, terraform.LogLevelDebug, + t.Context(), tc.tf, fs, io.Discard, terraform.LogLevelDebug, &terraform.QEMUVariables{}, templateDir, existingWorkspace, backupDir, ) diff --git a/cli/internal/cloudcmd/tfvars.go b/cli/internal/cloudcmd/tfvars.go index 4b78f7df8..0bcbb690e 100644 --- a/cli/internal/cloudcmd/tfvars.go +++ b/cli/internal/cloudcmd/tfvars.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd @@ -64,14 +64,6 @@ func TerraformIAMUpgradeVars(conf *config.Config, fileHandler file.Handler) (ter if err := terraform.VariablesFromBytes(oldVarBytes, &oldVars); err != nil { return nil, fmt.Errorf("parsing existing IAM workspace: %w", err) } - - // Migration from the "region" to the "location" field na. - // TODO(msanft): Remove after v2.14.0 is released. - if oldVars.Region != nil && *oldVars.Region != "" && oldVars.Location == "" { - oldVars.Location = *oldVars.Region - oldVars.Region = nil - } - vars = azureTerraformIAMVars(conf, oldVars) case cloudprovider.GCP: var oldVars terraform.GCPIAMVariables @@ -99,6 +91,7 @@ func awsTerraformVars(conf *config.Config, imageRef string) *terraform.AWSCluste DiskType: group.StateDiskType, } } + return &terraform.AWSClusterVariables{ Name: conf.Name, NodeGroups: nodeGroups, @@ -111,6 +104,7 @@ func awsTerraformVars(conf *config.Config, imageRef string) *terraform.AWSCluste EnableSNP: conf.GetAttestationConfig().GetVariant().Equal(variant.AWSSEVSNP{}), CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, + AdditionalTags: conf.Tags, } } @@ -153,18 +147,20 @@ func azureTerraformVars(conf *config.Config, imageRef string) (*terraform.AzureC } } vars := &terraform.AzureClusterVariables{ + SubscriptionID: conf.Provider.Azure.SubscriptionID, Name: conf.Name, NodeGroups: nodeGroups, Location: conf.Provider.Azure.Location, CreateMAA: toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})), Debug: toPtr(conf.IsDebugCluster()), - ConfidentialVM: toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})), + ConfidentialVM: toPtr(!conf.GetAttestationConfig().GetVariant().Equal(variant.AzureTrustedLaunch{})), SecureBoot: conf.Provider.Azure.SecureBoot, UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity, ResourceGroup: conf.Provider.Azure.ResourceGroup, CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, MarketplaceImage: nil, + AdditionalTags: conf.Tags, } if conf.UseMarketplaceImage() { @@ -196,6 +192,7 @@ func azureTerraformVars(conf *config.Config, imageRef string) (*terraform.AzureC func azureTerraformIAMVars(conf *config.Config, oldVars terraform.AzureIAMVariables) *terraform.AzureIAMVariables { return &terraform.AzureIAMVariables{ + SubscriptionID: conf.Provider.Azure.SubscriptionID, Location: conf.Provider.Azure.Location, ServicePrincipal: oldVars.ServicePrincipal, ResourceGroup: conf.Provider.Azure.ResourceGroup, @@ -216,6 +213,12 @@ func gcpTerraformVars(conf *config.Config, imageRef string) *terraform.GCPCluste DiskType: group.StateDiskType, } } + + ccTech := "SEV" + if conf.GetAttestationConfig().GetVariant().Equal(variant.GCPSEVSNP{}) { + ccTech = "SEV_SNP" + } + return &terraform.GCPClusterVariables{ Name: conf.Name, NodeGroups: nodeGroups, @@ -226,6 +229,9 @@ func gcpTerraformVars(conf *config.Config, imageRef string) *terraform.GCPCluste Debug: conf.IsDebugCluster(), CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, + CCTechnology: ccTech, + AdditionalLabels: conf.Tags, + IAMServiceAccountVM: conf.Provider.GCP.IAMServiceAccountVM, } } @@ -235,15 +241,13 @@ func gcpTerraformIAMVars(conf *config.Config, oldVars terraform.GCPIAMVariables) Region: conf.Provider.GCP.Region, Zone: conf.Provider.GCP.Zone, ServiceAccountID: oldVars.ServiceAccountID, + NamePrefix: oldVars.NamePrefix, } } // openStackTerraformVars provides variables required to execute the Terraform scripts. // It should be the only place to declare the OpenStack variables. func openStackTerraformVars(conf *config.Config, imageRef string) (*terraform.OpenStackClusterVariables, error) { - if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" { - return nil, errors.New("Constellation must be fine-tuned to your OpenStack deployment. Please create an issue or contact Edgeless Systems at https://edgeless.systems/contact/") - } if _, hasOSAuthURL := os.LookupEnv("OS_AUTH_URL"); !hasOSAuthURL && conf.Provider.OpenStack.Cloud == "" { return nil, errors.New( "neither environment variable OS_AUTH_URL nor cloud name for \"clouds.yaml\" is set. OpenStack authentication requires a set of " + @@ -264,19 +268,26 @@ func openStackTerraformVars(conf *config.Config, imageRef string) (*terraform.Op StateDiskType: group.StateDiskType, } } + + // since openstack does not support tags in the form of key = value, the tags will be converted + // to an array of "key=value" strings + tags := []string{} + for key, value := range conf.Tags { + tags = append(tags, fmt.Sprintf("%s=%s", key, value)) + } + return &terraform.OpenStackClusterVariables{ Name: conf.Name, Cloud: toPtr(conf.Provider.OpenStack.Cloud), + OpenStackCloudsYAMLPath: conf.Provider.OpenStack.CloudsYAMLPath, FloatingIPPoolID: conf.Provider.OpenStack.FloatingIPPoolID, - ImageURL: imageRef, - DirectDownload: *conf.Provider.OpenStack.DirectDownload, - OpenstackUserDomainName: conf.Provider.OpenStack.UserDomainName, - OpenstackUsername: conf.Provider.OpenStack.Username, - OpenstackPassword: conf.Provider.OpenStack.Password, + ImageID: imageRef, Debug: conf.IsDebugCluster(), NodeGroups: nodeGroups, CustomEndpoint: conf.CustomEndpoint, InternalLoadBalancer: conf.InternalLoadBalancer, + STACKITProjectID: conf.Provider.OpenStack.STACKITProjectID, + AdditionalTags: tags, }, nil } @@ -356,7 +367,7 @@ func qemuTerraformVars( ImagePath: imagePath, ImageFormat: conf.Provider.QEMU.ImageFormat, NodeGroups: nodeGroups, - Machine: "q35", // TODO(elchead): make configurable AB#3225 + Machine: "q35", MetadataAPIImage: conf.Provider.QEMU.MetadataAPIImage, MetadataLibvirtURI: metadataLibvirtURI, NVRAM: conf.Provider.QEMU.NVRAM, diff --git a/cli/internal/cloudcmd/tfvars_test.go b/cli/internal/cloudcmd/tfvars_test.go index 1a6b2a875..dc249bee0 100644 --- a/cli/internal/cloudcmd/tfvars_test.go +++ b/cli/internal/cloudcmd/tfvars_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudcmd diff --git a/cli/internal/cmd/BUILD.bazel b/cli/internal/cmd/BUILD.bazel index 5654b1dee..bc6a71a50 100644 --- a/cli/internal/cmd/BUILD.bazel +++ b/cli/internal/cmd/BUILD.bazel @@ -37,6 +37,7 @@ go_library( "miniup_linux_amd64.go", "recover.go", "spinner.go", + "ssh.go", "status.go", "terminate.go", "upgrade.go", @@ -97,6 +98,7 @@ go_library( "@com_github_google_uuid//:uuid", "@com_github_mattn_go_isatty//:go-isatty", "@com_github_rogpeppe_go_internal//diff", + "@com_github_samber_slog_multi//:slog-multi", "@com_github_siderolabs_talos_pkg_machinery//config/encoder", "@com_github_spf13_afero//:afero", "@com_github_spf13_cobra//:cobra", @@ -108,9 +110,15 @@ go_library( "@io_k8s_client_go//tools/clientcmd", "@io_k8s_client_go//tools/clientcmd/api/latest", "@io_k8s_sigs_yaml//:yaml", - "@org_golang_google_grpc//:go_default_library", "@org_golang_x_mod//semver", - "@org_uber_go_zap//zapcore", + "@org_golang_google_grpc//:grpc", + "@com_github_google_go_tdx_guest//abi", + "@com_github_google_go_tdx_guest//proto/tdx", + "//internal/attestation/azure/tdx", + "@com_github_google_go_sev_guest//proto/sevsnp", + "@com_github_google_go_tpm_tools//proto/attest", + "@org_golang_x_crypto//ssh", + "//internal/kms/setup", ] + select({ "@io_bazel_rules_go//go/platform:android_amd64": [ "@org_golang_x_sys//unix", @@ -137,6 +145,7 @@ go_test( "maapatch_test.go", "recover_test.go", "spinner_test.go", + "ssh_test.go", "status_test.go", "terminate_test.go", "upgradeapply_test.go", @@ -193,9 +202,10 @@ go_test( "@io_k8s_apimachinery//pkg/runtime/schema", "@io_k8s_client_go//tools/clientcmd", "@io_k8s_client_go//tools/clientcmd/api", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", + "@org_golang_x_crypto//ssh", "@org_golang_x_mod//semver", "@org_uber_go_goleak//:goleak", ], diff --git a/cli/internal/cmd/apply.go b/cli/internal/cmd/apply.go index e5c8caeac..d65337a0c 100644 --- a/cli/internal/cmd/apply.go +++ b/cli/internal/cmd/apply.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -13,7 +13,8 @@ import ( "fmt" "io" "io/fs" - "net" + "log/slog" + "os" "path/filepath" "slices" "strings" @@ -38,6 +39,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" + slogmulti "github.com/samber/slog-multi" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -208,11 +210,6 @@ func (f *applyFlags) parse(flags *pflag.FlagSet) error { // runApply sets up the apply command and runs it. func runApply(cmd *cobra.Command, _ []string) error { - log, err := newCLILogger(cmd) - if err != nil { - return fmt.Errorf("creating logger: %w", err) - } - defer log.Sync() spinner, err := newSpinnerOrStderr(cmd) if err != nil { return err @@ -225,9 +222,13 @@ func runApply(cmd *cobra.Command, _ []string) error { } fileHandler := file.NewHandler(afero.NewOsFs()) + debugLogger, err := newDebugFileLogger(cmd, fileHandler) + if err != nil { + return err + } newDialer := func(validator atls.Validator) *dialer.Dialer { - return dialer.New(nil, validator, &net.Dialer{}) + return dialer.New(nil, validator, nil) } upgradeID := generateUpgradeID(upgradeCmdKindApply) @@ -244,15 +245,15 @@ func runApply(cmd *cobra.Command, _ []string) error { ) } - applier := constellation.NewApplier(log, spinner, newDialer) + applier := constellation.NewApplier(debugLogger, spinner, constellation.ApplyContextCLI, newDialer) apply := &applyCmd{ fileHandler: fileHandler, flags: flags, - log: log, - wLog: &warnLogger{cmd: cmd, log: log}, + log: debugLogger, + wLog: &warnLogger{cmd: cmd, log: debugLogger}, spinner: spinner, - merger: &kubeconfigMerger{log: log}, + merger: &kubeconfigMerger{log: debugLogger}, newInfraApplier: newInfraApplier, imageFetcher: imagefetcher.New(), applier: applier, @@ -359,14 +360,14 @@ func (a *applyCmd) apply( } // Check license - a.checkLicenseFile(cmd, conf.GetProvider()) + a.checkLicenseFile(cmd, conf.GetProvider(), conf.UseMarketplaceImage()) // Now start actually running the apply command // Check current Terraform state, if it exists and infrastructure upgrades are not skipped, // and apply migrations if necessary. if !a.flags.skipPhases.contains(skipInfrastructurePhase) { - if err := a.runTerraformApply(cmd, conf, stateFile, upgradeDir); err != nil { + if err := a.runTerraformApply(cmd, conf, stateFile, upgradeDir, a.flags.yes); err != nil { return fmt.Errorf("applying Terraform configuration: %w", err) } } @@ -396,7 +397,7 @@ func (a *applyCmd) apply( // Apply Attestation Config if !a.flags.skipPhases.contains(skipAttestationConfigPhase) { - a.log.Debugf("Applying new attestation config to cluster") + a.log.Debug("Applying new attestation config to cluster") if err := a.applyJoinConfig(cmd, conf.GetAttestationConfig(), stateFile.ClusterValues.MeasurementSalt); err != nil { return fmt.Errorf("applying attestation config: %w", err) } @@ -416,9 +417,15 @@ func (a *applyCmd) apply( // Apply Helm Charts if !a.flags.skipPhases.contains(skipHelmPhase) { + if err := a.applier.AnnotateCoreDNSResources(cmd.Context()); err != nil { + return fmt.Errorf("annotating CoreDNS: %w", err) + } if err := a.runHelmApply(cmd, conf, stateFile, upgradeDir); err != nil { return err } + if err := a.applier.CleanupCoreDNSResources(cmd.Context()); err != nil { + return fmt.Errorf("cleaning up CoreDNS: %w", err) + } } // Upgrade node image @@ -443,7 +450,7 @@ func (a *applyCmd) apply( func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationconfigapi.Fetcher) (*config.Config, *state.State, error) { // Read user's config and state file - a.log.Debugf("Reading config from %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) + a.log.Debug(fmt.Sprintf("Reading config from %q", a.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) conf, err := config.New(a.fileHandler, constants.ConfigFilename, configFetcher, a.flags.force) var configValidationErr *config.ValidationError if errors.As(err, &configValidationErr) { @@ -453,7 +460,7 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc return nil, nil, err } - a.log.Debugf("Reading state file from %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename)) + a.log.Debug(fmt.Sprintf("Reading state file from %q", a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename))) stateFile, err := state.CreateOrRead(a.fileHandler, constants.StateFilename) if err != nil { return nil, nil, err @@ -464,16 +471,16 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc // We don't run "hard" verification of skip-phases flags and state file here, // a user may still end up skipping phases that could result in errors later on. // However, we perform basic steps, like ensuring init phase is not skipped if - a.log.Debugf("Validating state file") - preCreateValidateErr := stateFile.Validate(state.PreCreate, conf.GetProvider()) - preInitValidateErr := stateFile.Validate(state.PreInit, conf.GetProvider()) - postInitValidateErr := stateFile.Validate(state.PostInit, conf.GetProvider()) + a.log.Debug("Validating state file") + preCreateValidateErr := stateFile.Validate(state.PreCreate, conf.GetAttestationConfig().GetVariant()) + preInitValidateErr := stateFile.Validate(state.PreInit, conf.GetAttestationConfig().GetVariant()) + postInitValidateErr := stateFile.Validate(state.PostInit, conf.GetAttestationConfig().GetVariant()) // If the state file is in a pre-create state, we need to create the cluster, // in which case the workspace has to be clean if preCreateValidateErr == nil { // We can't skip the infrastructure phase if no infrastructure has been defined - a.log.Debugf("State file is in pre-create state, checking workspace") + a.log.Debug("State file is in pre-create state, checking workspace") if a.flags.skipPhases.contains(skipInfrastructurePhase) { return nil, nil, preInitValidateErr } @@ -482,7 +489,7 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc return nil, nil, err } - a.log.Debugf("No Terraform state found in current working directory. Preparing to create a new cluster.") + a.log.Debug("No Terraform state found in current working directory. Preparing to create a new cluster.") printCreateWarnings(cmd.ErrOrStderr(), conf) } @@ -491,7 +498,7 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc // If so, we need to run the init RPC if preInitValidateErr == nil || (preCreateValidateErr == nil && !a.flags.skipPhases.contains(skipInitPhase)) { // We can't skip the init phase if the init RPC hasn't been run yet - a.log.Debugf("State file is in pre-init state, checking workspace") + a.log.Debug("State file is in pre-init state, checking workspace") if a.flags.skipPhases.contains(skipInitPhase) { return nil, nil, postInitValidateErr } @@ -507,7 +514,7 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc // If the state file is in a post-init state, // we need to make sure specific files exist in the workspace if postInitValidateErr == nil { - a.log.Debugf("State file is in post-init state, checking workspace") + a.log.Debug("State file is in post-init state, checking workspace") if err := a.checkPostInitFilesExist(); err != nil { return nil, nil, err } @@ -522,16 +529,16 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc // If we need to run the init RPC, the version has to be valid // Otherwise, we are able to use an outdated version, meaning we skip the K8s upgrade // We skip version validation if the user explicitly skips the Kubernetes phase - a.log.Debugf("Validating Kubernetes version %s", conf.KubernetesVersion) + a.log.Debug(fmt.Sprintf("Validating Kubernetes version %q", conf.KubernetesVersion)) validVersion, err := versions.NewValidK8sVersion(string(conf.KubernetesVersion), true) if err != nil { - a.log.Debugf("Kubernetes version not valid: %s", err) + a.log.Debug(fmt.Sprintf("Kubernetes version not valid: %q", err)) if !a.flags.skipPhases.contains(skipInitPhase) { return nil, nil, err } if !a.flags.skipPhases.contains(skipK8sPhase) { - a.log.Debugf("Checking if user wants to continue anyway") + a.log.Debug("Checking if user wants to continue anyway") if !a.flags.yes { confirmed, err := askToConfirm(cmd, fmt.Sprintf( @@ -548,7 +555,7 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc } a.flags.skipPhases.add(skipK8sPhase) - a.log.Debugf("Outdated Kubernetes version accepted, Kubernetes upgrade will be skipped") + a.log.Debug("Outdated Kubernetes version accepted, Kubernetes upgrade will be skipped") } validVersionString, err := versions.ResolveK8sPatchVersion(xsemver.MajorMinor(string(conf.KubernetesVersion))) @@ -564,7 +571,7 @@ func (a *applyCmd) validateInputs(cmd *cobra.Command, configFetcher attestationc cmd.PrintErrf("Warning: Constellation with Kubernetes %s is still in preview. Use only for evaluation purposes.\n", validVersion) } conf.KubernetesVersion = validVersion - a.log.Debugf("Target Kubernetes version set to %s", conf.KubernetesVersion) + a.log.Debug(fmt.Sprintf("Target Kubernetes version set to %q", conf.KubernetesVersion)) // Validate microservice version (helm versions) in the user's config matches the version of the CLI // This makes sure we catch potential errors early, not just after we already ran Terraform migrations or the init RPC @@ -592,9 +599,9 @@ func (a *applyCmd) applyJoinConfig(cmd *cobra.Command, newConfig config.Attestat ) error { clusterAttestationConfig, err := a.applier.GetClusterAttestationConfig(cmd.Context(), newConfig.GetVariant()) if err != nil { - a.log.Debugf("Getting cluster attestation config failed: %s", err) + a.log.Debug(fmt.Sprintf("Getting cluster attestation config failed: %q", err)) if k8serrors.IsNotFound(err) { - a.log.Debugf("Creating new join config") + a.log.Debug("Creating new join config") return a.applier.ApplyJoinConfig(cmd.Context(), newConfig, measurementSalt) } return fmt.Errorf("getting cluster attestation config: %w", err) @@ -606,7 +613,7 @@ func (a *applyCmd) applyJoinConfig(cmd *cobra.Command, newConfig config.Attestat return fmt.Errorf("comparing attestation configs: %w", err) } if equal { - a.log.Debugf("Current attestation config is equal to the new config, nothing to do") + a.log.Debug("Current attestation config is equal to the new config, nothing to do") return nil } @@ -685,7 +692,7 @@ func (a *applyCmd) checkCreateFilesClean() error { if err := a.checkInitFilesClean(); err != nil { return err } - a.log.Debugf("Checking Terraform state") + a.log.Debug("Checking Terraform state") if _, err := a.fileHandler.Stat(constants.TerraformWorkingDir); err == nil { return fmt.Errorf( "terraform state %q already exists in working directory, run 'constellation terminate' before creating a new cluster", @@ -700,7 +707,7 @@ func (a *applyCmd) checkCreateFilesClean() error { // checkInitFilesClean ensures that the workspace is clean before running the init RPC. func (a *applyCmd) checkInitFilesClean() error { - a.log.Debugf("Checking admin configuration file") + a.log.Debug("Checking admin configuration file") if _, err := a.fileHandler.Stat(constants.AdminConfFilename); err == nil { return fmt.Errorf( "file %q already exists in working directory, run 'constellation terminate' before creating a new cluster", @@ -709,7 +716,7 @@ func (a *applyCmd) checkInitFilesClean() error { } else if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("checking for %q: %w", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename), err) } - a.log.Debugf("Checking master secrets file") + a.log.Debug("Checking master secrets file") if _, err := a.fileHandler.Stat(constants.MasterSecretFilename); err == nil { return fmt.Errorf( "file %q already exists in working directory. Constellation won't overwrite previous master secrets. Move it somewhere or delete it before creating a new cluster", @@ -805,26 +812,27 @@ type warnLogger struct { log debugLog } -// Infof messages are reduced to debug messages, since we don't want +// Info messages are reduced to debug messages, since we don't want // the extra info when using the CLI without setting the debug flag. -func (wl warnLogger) Infof(fmtStr string, args ...any) { - wl.log.Debugf(fmtStr, args...) +func (wl warnLogger) Info(msg string, args ...any) { + wl.log.Debug(msg, args...) } -// Warnf prints a formatted warning from the validator. -func (wl warnLogger) Warnf(fmtStr string, args ...any) { - wl.cmd.PrintErrf("Warning: %s\n", fmt.Sprintf(fmtStr, args...)) +// Warn prints a formatted warning from the validator. +func (wl warnLogger) Warn(msg string, args ...any) { + wl.cmd.PrintErrf("Warning: %s %s\n", msg, fmt.Sprint(args...)) + wl.log.Debug(msg, args...) } type warnLog interface { - Warnf(format string, args ...any) - Infof(format string, args ...any) + Warn(msg string, args ...any) + Info(msg string, args ...any) } // applier is used to run the different phases of the apply command. type applier interface { SetKubeConfig(kubeConfig []byte) error - CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error) + CheckLicense(ctx context.Context, csp cloudprovider.Provider, initRequest bool, licenseID string) (int, error) // methods required by "init" @@ -837,8 +845,10 @@ type applier interface { // methods required to install/upgrade Helm charts + AnnotateCoreDNSResources(context.Context) error + CleanupCoreDNSResources(context.Context) error PrepareHelmCharts( - flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig, + flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) // methods to interact with Kubernetes @@ -859,3 +869,34 @@ type imageFetcher interface { image, region string, useMarketplaceImage bool, ) (string, error) } + +func newDebugFileLogger(cmd *cobra.Command, fileHandler file.Handler) (debugLog, error) { + logLvl := slog.LevelInfo + debugLog, err := cmd.Flags().GetBool("debug") + if err != nil { + return nil, err + } + if debugLog { + logLvl = slog.LevelDebug + } + + fileWriter := &fileWriter{ + fileHandler: fileHandler, + } + return slog.New( + slogmulti.Fanout( + slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{AddSource: true, Level: logLvl}), // first handler: stderr at log level + slog.NewJSONHandler(fileWriter, &slog.HandlerOptions{AddSource: true, Level: slog.LevelDebug}), // second handler: debug JSON log to file + ), + ), nil +} + +type fileWriter struct { + fileHandler file.Handler +} + +// Write satisfies the io.Writer interface by writing a message to file. +func (l *fileWriter) Write(msg []byte) (int, error) { + err := l.fileHandler.Write(constants.CLIDebugLogFile, msg, file.OptAppend) + return len(msg), err +} diff --git a/cli/internal/cmd/apply_test.go b/cli/internal/cmd/apply_test.go index a8e010a03..9df359668 100644 --- a/cli/internal/cmd/apply_test.go +++ b/cli/internal/cmd/apply_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -199,7 +199,7 @@ func TestBackupHelmCharts(t *testing.T) { log: logger.NewTest(t), } - err := a.backupHelmCharts(context.Background(), tc.helmApplier, tc.includesUpgrades, "") + err := a.backupHelmCharts(t.Context(), tc.helmApplier, tc.includesUpgrades, "") if tc.wantErr { assert.Error(err) return @@ -238,7 +238,7 @@ func TestSkipPhases(t *testing.T) { } func TestValidateInputs(t *testing.T) { - defaultConfig := func(csp cloudprovider.Provider) func(require *require.Assertions, fh file.Handler) { + defaultConfig := func(csp cloudprovider.Provider) func(require *require.Assertions, _ file.Handler) { return func(require *require.Assertions, fh file.Handler) { cfg := defaultConfigWithExpectedMeasurements(t, config.Default(), csp) @@ -256,12 +256,13 @@ func TestValidateInputs(t *testing.T) { ClientX509CertURL: "client_cert", })) cfg.Provider.GCP.ServiceAccountKeyPath = "saKey.json" + cfg.Provider.GCP.IAMServiceAccountVM = "example@example.com" } require.NoError(fh.WriteYAML(constants.ConfigFilename, cfg)) } } - preInitState := func(csp cloudprovider.Provider) func(require *require.Assertions, fh file.Handler) { + preInitState := func(csp cloudprovider.Provider) func(_ *require.Assertions, _ file.Handler) { return func(require *require.Assertions, fh file.Handler) { stateFile := defaultStateFile(csp) stateFile.ClusterValues = state.ClusterValues{} @@ -332,7 +333,7 @@ func TestValidateInputs(t *testing.T) { wantPhases: newPhases(skipInitPhase, skipImagePhase), // No image upgrades on QEMU }, "no config file errors": { - createConfig: func(require *require.Assertions, fh file.Handler) {}, + createConfig: func(_ *require.Assertions, _ file.Handler) {}, createState: postInitState(cloudprovider.GCP), createMasterSecret: defaultMasterSecret, createAdminConfig: defaultAdminConfig, @@ -344,7 +345,7 @@ func TestValidateInputs(t *testing.T) { createConfig: defaultConfig(cloudprovider.GCP), createState: preInitState(cloudprovider.GCP), createMasterSecret: defaultMasterSecret, - createAdminConfig: func(require *require.Assertions, fh file.Handler) {}, + createAdminConfig: func(_ *require.Assertions, _ file.Handler) {}, createTfState: defaultTfState, flags: applyFlags{}, wantErr: true, @@ -352,8 +353,8 @@ func TestValidateInputs(t *testing.T) { "[init] no admin config file, no master secret": { createConfig: defaultConfig(cloudprovider.GCP), createState: preInitState(cloudprovider.GCP), - createMasterSecret: func(require *require.Assertions, fh file.Handler) {}, - createAdminConfig: func(require *require.Assertions, fh file.Handler) {}, + createMasterSecret: func(_ *require.Assertions, _ file.Handler) {}, + createAdminConfig: func(_ *require.Assertions, _ file.Handler) {}, createTfState: defaultTfState, flags: applyFlags{}, wantPhases: newPhases(skipImagePhase, skipK8sPhase), @@ -363,16 +364,16 @@ func TestValidateInputs(t *testing.T) { createState: preInitState(cloudprovider.GCP), createMasterSecret: defaultMasterSecret, createAdminConfig: defaultAdminConfig, - createTfState: func(require *require.Assertions, fh file.Handler) {}, + createTfState: func(_ *require.Assertions, _ file.Handler) {}, flags: applyFlags{}, wantErr: true, }, "[create] only config, skip everything but infrastructure": { createConfig: defaultConfig(cloudprovider.GCP), - createState: func(require *require.Assertions, fh file.Handler) {}, - createMasterSecret: func(require *require.Assertions, fh file.Handler) {}, - createAdminConfig: func(require *require.Assertions, fh file.Handler) {}, - createTfState: func(require *require.Assertions, fh file.Handler) {}, + createState: func(_ *require.Assertions, _ file.Handler) {}, + createMasterSecret: func(_ *require.Assertions, _ file.Handler) {}, + createAdminConfig: func(_ *require.Assertions, _ file.Handler) {}, + createTfState: func(_ *require.Assertions, _ file.Handler) {}, flags: applyFlags{ skipPhases: newPhases(skipInitPhase, skipAttestationConfigPhase, skipCertSANsPhase, skipHelmPhase, skipK8sPhase, skipImagePhase), }, @@ -380,19 +381,19 @@ func TestValidateInputs(t *testing.T) { }, "[create + init] only config file": { createConfig: defaultConfig(cloudprovider.GCP), - createState: func(require *require.Assertions, fh file.Handler) {}, - createMasterSecret: func(require *require.Assertions, fh file.Handler) {}, - createAdminConfig: func(require *require.Assertions, fh file.Handler) {}, - createTfState: func(require *require.Assertions, fh file.Handler) {}, + createState: func(_ *require.Assertions, _ file.Handler) {}, + createMasterSecret: func(_ *require.Assertions, _ file.Handler) {}, + createAdminConfig: func(_ *require.Assertions, _ file.Handler) {}, + createTfState: func(_ *require.Assertions, _ file.Handler) {}, flags: applyFlags{}, wantPhases: newPhases(skipImagePhase, skipK8sPhase), }, "[init] self-managed: config and state file exist, skip-phases=infrastructure": { createConfig: defaultConfig(cloudprovider.GCP), createState: preInitState(cloudprovider.GCP), - createMasterSecret: func(require *require.Assertions, fh file.Handler) {}, - createAdminConfig: func(require *require.Assertions, fh file.Handler) {}, - createTfState: func(require *require.Assertions, fh file.Handler) {}, + createMasterSecret: func(_ *require.Assertions, _ file.Handler) {}, + createAdminConfig: func(_ *require.Assertions, _ file.Handler) {}, + createTfState: func(_ *require.Assertions, _ file.Handler) {}, flags: applyFlags{ skipPhases: newPhases(skipInfrastructurePhase), }, @@ -414,7 +415,7 @@ func TestValidateInputs(t *testing.T) { createTfState: defaultTfState, stdin: "y\n", wantPhases: newPhases(skipInitPhase, skipK8sPhase), - assert: func(require *require.Assertions, assert *assert.Assertions, conf *config.Config, stateFile *state.State) { + assert: func(_ *require.Assertions, assert *assert.Assertions, conf *config.Config, _ *state.State) { assert.NotEmpty(conf.KubernetesVersion) _, err := versions.NewValidK8sVersion(string(conf.KubernetesVersion), true) assert.NoError(err) @@ -536,7 +537,7 @@ type stubConstellApplier struct { func (s *stubConstellApplier) SetKubeConfig([]byte) error { return nil } -func (s *stubConstellApplier) CheckLicense(context.Context, cloudprovider.Provider, string) (int, error) { +func (s *stubConstellApplier) CheckLicense(context.Context, cloudprovider.Provider, bool, string) (int, error) { return 0, s.checkLicenseErr } @@ -553,8 +554,10 @@ func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State } type helmApplier interface { + AnnotateCoreDNSResources(context.Context) error + CleanupCoreDNSResources(ctx context.Context) error PrepareHelmCharts( - flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig, + flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) ( helm.Applier, bool, error) } diff --git a/cli/internal/cmd/applyhelm.go b/cli/internal/cmd/applyhelm.go index 74e65ff5a..2626da306 100644 --- a/cli/internal/cmd/applyhelm.go +++ b/cli/internal/cmd/applyhelm.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -25,7 +25,7 @@ import ( // runHelmApply handles installing or upgrading helm charts for the cluster. func (a *applyCmd) runHelmApply(cmd *cobra.Command, conf *config.Config, stateFile *state.State, upgradeDir string, ) error { - a.log.Debugf("Installing or upgrading Helm charts") + a.log.Debug("Installing or upgrading Helm charts") var masterSecret uri.MasterSecret if err := a.fileHandler.ReadJSON(constants.MasterSecretFilename, &masterSecret); err != nil { return fmt.Errorf("reading master secret: %w", err) @@ -42,16 +42,29 @@ func (a *applyCmd) runHelmApply(cmd *cobra.Command, conf *config.Config, stateFi HelmWaitMode: a.flags.helmWaitMode, ApplyTimeout: a.flags.helmTimeout, AllowDestructive: helm.DenyDestructive, + ServiceCIDR: conf.ServiceCIDR, + } + if conf.Provider.OpenStack != nil { + var deployYawolLoadBalancer bool + if conf.Provider.OpenStack.DeployYawolLoadBalancer != nil { + deployYawolLoadBalancer = *conf.Provider.OpenStack.DeployYawolLoadBalancer + } + options.OpenStackValues = &helm.OpenStackValues{ + DeployYawolLoadBalancer: deployYawolLoadBalancer, + FloatingIPPoolID: conf.Provider.OpenStack.FloatingIPPoolID, + YawolFlavorID: conf.Provider.OpenStack.YawolFlavorID, + YawolImageID: conf.Provider.OpenStack.YawolImageID, + } } - a.log.Debugf("Getting service account URI") + a.log.Debug("Getting service account URI") serviceAccURI, err := cloudcmd.GetMarshaledServiceAccountURI(conf, a.fileHandler) if err != nil { return err } - a.log.Debugf("Preparing Helm charts") - executor, includesUpgrades, err := a.applier.PrepareHelmCharts(options, stateFile, serviceAccURI, masterSecret, conf.Provider.OpenStack) + a.log.Debug("Preparing Helm charts") + executor, includesUpgrades, err := a.applier.PrepareHelmCharts(options, stateFile, serviceAccURI, masterSecret) if errors.Is(err, helm.ErrConfirmationMissing) { if !a.flags.yes { cmd.PrintErrln("WARNING: Upgrading cert-manager will destroy all custom resources you have manually created that are based on the current version of cert-manager.") @@ -65,7 +78,7 @@ func (a *applyCmd) runHelmApply(cmd *cobra.Command, conf *config.Config, stateFi } } options.AllowDestructive = helm.AllowDestructive - executor, includesUpgrades, err = a.applier.PrepareHelmCharts(options, stateFile, serviceAccURI, masterSecret, conf.Provider.OpenStack) + executor, includesUpgrades, err = a.applier.PrepareHelmCharts(options, stateFile, serviceAccURI, masterSecret) } var upgradeErr *compatibility.InvalidUpgradeError if err != nil { @@ -75,12 +88,12 @@ func (a *applyCmd) runHelmApply(cmd *cobra.Command, conf *config.Config, stateFi cmd.PrintErrln(err) } - a.log.Debugf("Backing up Helm charts") + a.log.Debug("Backing up Helm charts") if err := a.backupHelmCharts(cmd.Context(), executor, includesUpgrades, upgradeDir); err != nil { return err } - a.log.Debugf("Applying Helm charts") + a.log.Debug("Applying Helm charts") if !a.flags.skipPhases.contains(skipInitPhase) { a.spinner.Start("Installing Kubernetes components ", false) } else { @@ -108,10 +121,10 @@ func (a *applyCmd) backupHelmCharts( if err := executor.SaveCharts(chartDir, a.fileHandler); err != nil { return fmt.Errorf("saving Helm charts to disk: %w", err) } - a.log.Debugf("Helm charts saved to %s", a.flags.pathPrefixer.PrefixPrintablePath(chartDir)) + a.log.Debug(fmt.Sprintf("Helm charts saved to %q", a.flags.pathPrefixer.PrefixPrintablePath(chartDir))) if includesUpgrades { - a.log.Debugf("Creating backup of CRDs and CRs") + a.log.Debug("Creating backup of CRDs and CRs") crds, err := a.applier.BackupCRDs(ctx, a.fileHandler, upgradeDir) if err != nil { return fmt.Errorf("creating CRD backup: %w", err) diff --git a/cli/internal/cmd/applyinit.go b/cli/internal/cmd/applyinit.go index 90fa77cdc..2bdaa32f8 100644 --- a/cli/internal/cmd/applyinit.go +++ b/cli/internal/cmd/applyinit.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -29,13 +29,13 @@ import ( // On success, it writes the Kubernetes admin config file to disk. // Therefore it is skipped if the Kubernetes admin config file already exists. func (a *applyCmd) runInit(cmd *cobra.Command, conf *config.Config, stateFile *state.State) (*bytes.Buffer, error) { - a.log.Debugf("Creating aTLS Validator for %s", conf.GetAttestationConfig().GetVariant()) + a.log.Debug(fmt.Sprintf("Creating aTLS Validator for %q", conf.GetAttestationConfig().GetVariant())) validator, err := choose.Validator(conf.GetAttestationConfig(), a.wLog) if err != nil { return nil, fmt.Errorf("creating validator: %w", err) } - a.log.Debugf("Running init RPC") + a.log.Debug("Running init RPC") masterSecret, err := a.generateAndPersistMasterSecret(cmd.OutOrStdout()) if err != nil { return nil, fmt.Errorf("generating master secret: %w", err) @@ -74,9 +74,9 @@ func (a *applyCmd) runInit(cmd *cobra.Command, conf *config.Config, stateFile *s } return nil, err } - a.log.Debugf("Initialization request successful") + a.log.Debug("Initialization request successful") - a.log.Debugf("Buffering init success message") + a.log.Debug("Buffering init success message") bufferedOutput := &bytes.Buffer{} if err := a.writeInitOutput(stateFile, resp, a.flags.mergeConfigs, bufferedOutput, measurementSalt); err != nil { return nil, err @@ -121,7 +121,7 @@ func (a *applyCmd) writeInitOutput( if err := a.fileHandler.Write(constants.AdminConfFilename, initResp.Kubeconfig, file.OptNone); err != nil { return fmt.Errorf("writing kubeconfig: %w", err) } - a.log.Debugf("Kubeconfig written to %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) + a.log.Debug(fmt.Sprintf("Kubeconfig written to %q", a.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename))) if mergeConfig { if err := a.merger.mergeConfigs(constants.AdminConfFilename, a.fileHandler); err != nil { @@ -136,7 +136,7 @@ func (a *applyCmd) writeInitOutput( return fmt.Errorf("writing Constellation state file: %w", err) } - a.log.Debugf("Constellation state file written to %s", a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename)) + a.log.Debug(fmt.Sprintf("Constellation state file written to %q", a.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename))) if !mergeConfig { fmt.Fprintln(wr, "You can now connect to your cluster by executing:") diff --git a/cli/internal/cmd/applyterraform.go b/cli/internal/cmd/applyterraform.go index fe65d8627..14cba64e4 100644 --- a/cli/internal/cmd/applyterraform.go +++ b/cli/internal/cmd/applyterraform.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -13,6 +13,7 @@ import ( "path/filepath" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -21,8 +22,8 @@ import ( ) // runTerraformApply checks if changes to Terraform are required and applies them. -func (a *applyCmd) runTerraformApply(cmd *cobra.Command, conf *config.Config, stateFile *state.State, upgradeDir string) error { - a.log.Debugf("Checking if Terraform migrations are required") +func (a *applyCmd) runTerraformApply(cmd *cobra.Command, conf *config.Config, stateFile *state.State, upgradeDir string, yesFlag bool) error { + a.log.Debug("Checking if Terraform migrations are required") terraformClient, removeClient, err := a.newInfraApplier(cmd.Context()) if err != nil { return fmt.Errorf("creating Terraform client: %w", err) @@ -35,21 +36,35 @@ func (a *applyCmd) runTerraformApply(cmd *cobra.Command, conf *config.Config, st return fmt.Errorf("checking if Terraform workspace is empty: %w", err) } + if !isNewCluster && cloudcmd.UpgradeRequiresIAMMigration(conf.GetProvider()) { + cmd.Println("WARNING: This upgrade requires an IAM migration. Please make sure you have applied the IAM migration using `iam upgrade apply` before continuing.") + if !yesFlag { + yes, err := askToConfirm(cmd, "Did you upgrade the IAM resources?") + if err != nil { + return fmt.Errorf("asking for confirmation: %w", err) + } + if !yes { + cmd.Println("Skipping upgrade.") + return nil + } + } + } + if changesRequired, err := a.planTerraformChanges(cmd, conf, terraformClient); err != nil { return fmt.Errorf("planning Terraform migrations: %w", err) } else if !changesRequired { - a.log.Debugf("No changes to infrastructure required, skipping Terraform migrations") + a.log.Debug("No changes to infrastructure required, skipping Terraform migrations") return nil } - a.log.Debugf("Apply new Terraform resources for infrastructure changes") + a.log.Debug("Apply new Terraform resources for infrastructure changes") newInfraState, err := a.applyTerraformChanges(cmd, conf, terraformClient, upgradeDir, isNewCluster) if err != nil { return err } // Merge the original state with the new infrastructure values - a.log.Debugf("Updating state file with new infrastructure state") + a.log.Debug("Updating state file with new infrastructure state") if _, err := stateFile.Merge( // temporary state with new infrastructure values state.New().SetInfrastructure(newInfraState), @@ -67,7 +82,7 @@ func (a *applyCmd) runTerraformApply(cmd *cobra.Command, conf *config.Config, st // planTerraformChanges checks if any changes to the Terraform state are required. // If no state exists, this function will return true and the caller should create a new state. func (a *applyCmd) planTerraformChanges(cmd *cobra.Command, conf *config.Config, terraformClient cloudApplier) (bool, error) { - a.log.Debugf("Planning Terraform changes") + a.log.Debug("Planning Terraform changes") // Check if there are any Terraform changes to apply @@ -75,7 +90,7 @@ func (a *applyCmd) planTerraformChanges(cmd *cobra.Command, conf *config.Config, // // var manualMigrations []terraform.StateMigration // for _, migration := range manualMigrations { - // u.log.Debugf("Adding manual Terraform migration: %s", migration.DisplayName) + // u.log.Debug(fmt.Sprintf("Adding manual Terraform migration: %s", migration.DisplayName)) // u.infraApplier.AddManualStateMigration(migration) // } @@ -94,7 +109,8 @@ func (a *applyCmd) applyTerraformChanges( return state.Infrastructure{}, err } return a.applyTerraformChangesWithMessage( - cmd, conf.GetProvider(), cloudcmd.WithRollbackOnError, terraformClient, upgradeDir, + cmd, conf.GetProvider(), conf.GetAttestationConfig().GetVariant(), + cloudcmd.WithRollbackOnError, terraformClient, upgradeDir, "Do you want to create this cluster?", "The creation of the cluster was aborted.", "cluster creation aborted by user", @@ -105,7 +121,8 @@ func (a *applyCmd) applyTerraformChanges( cmd.Println("Changes of Constellation cloud resources are required by applying an updated Terraform template.") return a.applyTerraformChangesWithMessage( - cmd, conf.GetProvider(), cloudcmd.WithoutRollbackOnError, terraformClient, upgradeDir, + cmd, conf.GetProvider(), conf.GetAttestationConfig().GetVariant(), + cloudcmd.WithoutRollbackOnError, terraformClient, upgradeDir, "Do you want to apply these Terraform changes?", "Aborting upgrade.", "cluster upgrade aborted by user", @@ -119,8 +136,8 @@ func (a *applyCmd) applyTerraformChanges( } func (a *applyCmd) applyTerraformChangesWithMessage( - cmd *cobra.Command, csp cloudprovider.Provider, rollbackBehavior cloudcmd.RollbackBehavior, - terraformClient cloudApplier, upgradeDir string, + cmd *cobra.Command, csp cloudprovider.Provider, attestation variant.Variant, + rollbackBehavior cloudcmd.RollbackBehavior, terraformClient cloudApplier, upgradeDir string, confirmationQst, abortMsg, abortErrorMsg, progressMsg, successMsg string, ) (state.Infrastructure, error) { // Ask for confirmation first @@ -143,10 +160,10 @@ func (a *applyCmd) applyTerraformChangesWithMessage( return state.Infrastructure{}, errors.New(abortErrorMsg) } } - a.log.Debugf("Applying Terraform changes") + a.log.Debug("Applying Terraform changes") a.spinner.Start(progressMsg, false) - infraState, err := terraformClient.Apply(cmd.Context(), csp, rollbackBehavior) + infraState, err := terraformClient.Apply(cmd.Context(), csp, attestation, rollbackBehavior) a.spinner.Stop() if err != nil { return state.Infrastructure{}, fmt.Errorf("applying terraform changes: %w", err) @@ -183,7 +200,7 @@ func printCreateInfo(out io.Writer, conf *config.Config, log debugLog) error { } } if len(otherGroupNames) > 0 { - log.Debugf("Creating %d additional node groups: %v", len(otherGroupNames), otherGroupNames) + log.Debug(fmt.Sprintf("Creating %d additional node groups: %v", len(otherGroupNames), otherGroupNames)) } fmt.Fprintf(out, "The following Constellation cluster will be created:\n") diff --git a/cli/internal/cmd/cloud.go b/cli/internal/cmd/cloud.go index 1783713e9..e16e2331e 100644 --- a/cli/internal/cmd/cloud.go +++ b/cli/internal/cmd/cloud.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -11,6 +11,7 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" @@ -19,7 +20,7 @@ import ( type cloudApplier interface { Plan(ctx context.Context, conf *config.Config) (bool, error) - Apply(ctx context.Context, csp cloudprovider.Provider, rollback cloudcmd.RollbackBehavior) (state.Infrastructure, error) + Apply(ctx context.Context, csp cloudprovider.Provider, variant variant.Variant, rollback cloudcmd.RollbackBehavior) (state.Infrastructure, error) RestoreWorkspace() error WorkingDirIsEmpty() (bool, error) } diff --git a/cli/internal/cmd/cloud_test.go b/cli/internal/cmd/cloud_test.go index ceb8636a5..20ba89808 100644 --- a/cli/internal/cmd/cloud_test.go +++ b/cli/internal/cmd/cloud_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/cli/internal/terraform" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/config" @@ -23,6 +24,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -43,7 +45,7 @@ func (c *stubCloudCreator) Plan(_ context.Context, _ *config.Config) (bool, erro return c.planDiff, c.planErr } -func (c *stubCloudCreator) Apply(_ context.Context, _ cloudprovider.Provider, _ cloudcmd.RollbackBehavior) (state.Infrastructure, error) { +func (c *stubCloudCreator) Apply(_ context.Context, _ cloudprovider.Provider, _ variant.Variant, _ cloudcmd.RollbackBehavior) (state.Infrastructure, error) { c.applyCalled = true return c.state, c.applyErr } diff --git a/cli/internal/cmd/cmd.go b/cli/internal/cmd/cmd.go index a5997b941..69b542d89 100644 --- a/cli/internal/cmd/cmd.go +++ b/cli/internal/cmd/cmd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/cli/internal/cmd/config.go b/cli/internal/cmd/config.go index 3392df473..9386a8838 100644 --- a/cli/internal/cmd/config.go +++ b/cli/internal/cmd/config.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/configfetchmeasurements.go b/cli/internal/cmd/configfetchmeasurements.go index 14dd11c87..a9b777973 100644 --- a/cli/internal/cmd/configfetchmeasurements.go +++ b/cli/internal/cmd/configfetchmeasurements.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -93,7 +93,6 @@ func runConfigFetchMeasurements(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() fileHandler := file.NewHandler(afero.NewOsFs()) rekor, err := sigstore.NewRekor() if err != nil { @@ -105,7 +104,7 @@ func runConfigFetchMeasurements(cmd *cobra.Command, _ []string) error { if err := cfm.flags.parse(cmd.Flags()); err != nil { return fmt.Errorf("parsing flags: %w", err) } - cfm.log.Debugf("Using flags %+v", cfm.flags) + cfm.log.Debug("Using flags", "insecure", cfm.flags.insecure, "measurementsURL", cfm.flags.measurementsURL, "signatureURL", cfm.flags.signatureURL) fetcher := attestationconfigapi.NewFetcherWithClient(http.DefaultClient, constants.CDNRepositoryURL) return cfm.configFetchMeasurements(cmd, fileHandler, fetcher) @@ -119,7 +118,7 @@ func (cfm *configFetchMeasurementsCmd) configFetchMeasurements( return errors.New("fetching measurements is not supported") } - cfm.log.Debugf("Loading configuration file from %q", cfm.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) + cfm.log.Debug(fmt.Sprintf("Loading configuration file from %q", cfm.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) conf, err := config.New(fileHandler, constants.ConfigFilename, fetcher, cfm.flags.force) var configValidationErr *config.ValidationError @@ -134,11 +133,11 @@ func (cfm *configFetchMeasurementsCmd) configFetchMeasurements( cmd.PrintErrln("Configured image doesn't look like a released production image. Double check image before deploying to production.") } - cfm.log.Debugf("Creating context") + cfm.log.Debug("Creating context") ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - cfm.log.Debugf("Updating URLs") + cfm.log.Debug("Updating URLs") if err := cfm.flags.updateURLs(conf); err != nil { return err } @@ -153,14 +152,14 @@ func (cfm *configFetchMeasurementsCmd) configFetchMeasurements( return fmt.Errorf("fetching and verifying measurements: %w", err) } } - cfm.log.Debugf("Measurements:\n", fetchedMeasurements) + cfm.log.Debug(fmt.Sprintf("Measurements: %s", fetchedMeasurements.String())) - cfm.log.Debugf("Updating measurements in configuration") + cfm.log.Debug("Updating measurements in configuration") conf.UpdateMeasurements(fetchedMeasurements) if err := fileHandler.WriteYAML(constants.ConfigFilename, conf, file.OptOverwrite); err != nil { return err } - cfm.log.Debugf("Configuration written to %s", cfm.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) + cfm.log.Debug(fmt.Sprintf("Configuration written to %q", cfm.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) cmd.Print("Successfully fetched measurements and updated Configuration\n") return nil } diff --git a/cli/internal/cmd/configfetchmeasurements_test.go b/cli/internal/cmd/configfetchmeasurements_test.go index 9cebbb7da..5eca25c1c 100644 --- a/cli/internal/cmd/configfetchmeasurements_test.go +++ b/cli/internal/cmd/configfetchmeasurements_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -204,18 +204,8 @@ func (f stubVerifyFetcher) FetchAndVerifyMeasurements(_ context.Context, _ strin type stubAttestationFetcher struct{} -func (f stubAttestationFetcher) FetchSEVSNPVersionList(_ context.Context, _ attestationconfigapi.SEVSNPVersionList) (attestationconfigapi.SEVSNPVersionList, error) { - return attestationconfigapi.SEVSNPVersionList{}, nil -} - -func (f stubAttestationFetcher) FetchSEVSNPVersion(_ context.Context, _ attestationconfigapi.SEVSNPVersionAPI) (attestationconfigapi.SEVSNPVersionAPI, error) { - return attestationconfigapi.SEVSNPVersionAPI{ - SEVSNPVersion: testCfg, - }, nil -} - -func (f stubAttestationFetcher) FetchSEVSNPVersionLatest(_ context.Context, _ variant.Variant) (attestationconfigapi.SEVSNPVersionAPI, error) { - return attestationconfigapi.SEVSNPVersionAPI{ +func (f stubAttestationFetcher) FetchLatestVersion(_ context.Context, _ variant.Variant) (attestationconfigapi.Entry, error) { + return attestationconfigapi.Entry{ SEVSNPVersion: testCfg, }, nil } diff --git a/cli/internal/cmd/configgenerate.go b/cli/internal/cmd/configgenerate.go index d2268cfef..f5cc297e4 100644 --- a/cli/internal/cmd/configgenerate.go +++ b/cli/internal/cmd/configgenerate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -37,6 +37,7 @@ func newConfigGenerateCmd() *cobra.Command { } cmd.Flags().StringP("kubernetes", "k", semver.MajorMinor(string(config.Default().KubernetesVersion)), "Kubernetes version to use in format MAJOR.MINOR") cmd.Flags().StringP("attestation", "a", "", fmt.Sprintf("attestation variant to use %s. If not specified, the default for the cloud provider is used", printFormattedSlice(variant.GetAvailableAttestationVariants()))) + cmd.Flags().StringSliceP("tags", "t", nil, "additional tags for created resources given a list of key=value") return cmd } @@ -45,6 +46,7 @@ type generateFlags struct { rootFlags k8sVersion versions.ValidK8sVersion attestationVariant variant.Variant + tags cloudprovider.Tags } func (f *generateFlags) parse(flags *pflag.FlagSet) error { @@ -64,6 +66,12 @@ func (f *generateFlags) parse(flags *pflag.FlagSet) error { } f.attestationVariant = variant + tags, err := parseTagsFlags(flags) + if err != nil { + return err + } + f.tags = tags + return nil } @@ -77,7 +85,6 @@ func runConfigGenerate(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() fileHandler := file.NewHandler(afero.NewOsFs()) provider := cloudprovider.FromString(args[0]) @@ -86,13 +93,13 @@ func runConfigGenerate(cmd *cobra.Command, args []string) error { if err := cg.flags.parse(cmd.Flags()); err != nil { return fmt.Errorf("parsing flags: %w", err) } - log.Debugf("Parsed flags as %+v", cg.flags) + log.Debug("Using flags", "k8sVersion", cg.flags.k8sVersion, "attestationVariant", cg.flags.attestationVariant) return cg.configGenerate(cmd, fileHandler, provider, args[0]) } func (cg *configGenerateCmd) configGenerate(cmd *cobra.Command, fileHandler file.Handler, provider cloudprovider.Provider, rawProvider string) error { - cg.log.Debugf("Using cloud provider %s", provider.String()) + cg.log.Debug(fmt.Sprintf("Using cloud provider %q", provider.String())) // Config creation conf, err := createConfigWithAttestationVariant(provider, rawProvider, cg.flags.attestationVariant) @@ -100,7 +107,8 @@ func (cg *configGenerateCmd) configGenerate(cmd *cobra.Command, fileHandler file return fmt.Errorf("creating config: %w", err) } conf.KubernetesVersion = cg.flags.k8sVersion - cg.log.Debugf("Writing YAML data to configuration file") + conf.Tags = cg.flags.tags + cg.log.Debug("Writing YAML data to configuration file") if err := fileHandler.WriteYAML(constants.ConfigFilename, conf, file.OptMkdirAll); err != nil { return fmt.Errorf("writing config file: %w", err) } @@ -129,7 +137,7 @@ func (cg *configGenerateCmd) configGenerate(cmd *cobra.Command, fileHandler file // createConfigWithAttestationVariant creates a config file for the given provider. func createConfigWithAttestationVariant(provider cloudprovider.Provider, rawProvider string, attestationVariant variant.Variant) (*config.Config, error) { - conf := config.Default().WithOpenStackProviderDefaults(rawProvider) + conf := config.Default().WithOpenStackProviderDefaults(provider, rawProvider) conf.RemoveProviderExcept(provider) // set a lower default for QEMU's state disk @@ -152,6 +160,8 @@ func createConfigWithAttestationVariant(provider cloudprovider.Provider, rawProv return nil, fmt.Errorf("provider %s does not support attestation variant %s", provider, attestationVariant) } conf.SetAttestation(attestationVariant) + + conf.SetCSPNodeGroupDefaults(provider) return conf, nil } @@ -220,3 +230,27 @@ func parseAttestationFlag(flags *pflag.FlagSet) (variant.Variant, error) { return attestationVariant, nil } + +func parseTagsFlags(flags *pflag.FlagSet) (cloudprovider.Tags, error) { + tagsSlice, err := flags.GetStringSlice("tags") + if err != nil { + return nil, fmt.Errorf("getting tags flag: %w", err) + } + + // no tags given + if tagsSlice == nil { + return nil, nil + } + + tags := make(cloudprovider.Tags) + for _, tag := range tagsSlice { + tagSplit := strings.Split(tag, "=") + if len(tagSplit) != 2 { + return nil, fmt.Errorf("wrong format of tags: expected \"key=value\", got %q", tag) + } + + tags[tagSplit[0]] = tagSplit[1] + } + + return tags, nil +} diff --git a/cli/internal/cmd/configgenerate_test.go b/cli/internal/cmd/configgenerate_test.go index d3ffbff18..cbb1349e9 100644 --- a/cli/internal/cmd/configgenerate_test.go +++ b/cli/internal/cmd/configgenerate_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -140,7 +140,7 @@ func TestConfigGenerateDefaultProviderSpecific(t *testing.T) { fileHandler := file.NewHandler(afero.NewMemMapFs()) cmd := newConfigGenerateCmd() - wantConf := config.Default().WithOpenStackProviderDefaults(tc.rawProvider) + wantConf := config.Default().WithOpenStackProviderDefaults(cloudprovider.OpenStack, tc.rawProvider) wantConf.RemoveProviderAndAttestationExcept(tc.provider) cg := &configGenerateCmd{ @@ -196,7 +196,7 @@ func TestNoValidProviderAttestationCombination(t *testing.T) { {cloudprovider.OpenStack, variant.AWSNitroTPM{}}, } for _, test := range tests { - t.Run("", func(t *testing.T) { + t.Run("", func(_ *testing.T) { _, err := createConfigWithAttestationVariant(test.provider, "", test.attestation) assert.Error(err) }) @@ -235,6 +235,11 @@ func TestValidProviderAttestationCombination(t *testing.T) { variant.GCPSEVES{}, config.AttestationConfig{GCPSEVES: defaultAttestation.GCPSEVES}, }, + { + cloudprovider.GCP, + variant.GCPSEVSNP{}, + config.AttestationConfig{GCPSEVSNP: defaultAttestation.GCPSEVSNP}, + }, { cloudprovider.QEMU, variant.QEMUVTPM{}, @@ -286,6 +291,10 @@ func TestParseAttestationFlag(t *testing.T) { attestationFlag: "gcp-sev-es", wantVariant: variant.GCPSEVES{}, }, + "GCPSEVSNP": { + attestationFlag: "gcp-sev-snp", + wantVariant: variant.GCPSEVSNP{}, + }, "QEMUVTPM": { attestationFlag: "qemu-vtpm", wantVariant: variant.QEMUVTPM{}, diff --git a/cli/internal/cmd/configinstancetypes.go b/cli/internal/cmd/configinstancetypes.go index 8d9a3591d..0c4c4a73d 100644 --- a/cli/internal/cmd/configinstancetypes.go +++ b/cli/internal/cmd/configinstancetypes.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -30,13 +30,25 @@ func printSupportedInstanceTypes(cmd *cobra.Command, _ []string) { %v AWS NitroTPM-enabled instance types: %v -Azure Confidential VM instance types: +Azure Intel TDX instance types: +%v +Azure AMD SEV-SNP instance types: %v Azure Trusted Launch instance types: %v GCP instance types: %v -`, formatInstanceTypes(instancetypes.AWSSNPSupportedInstanceFamilies), formatInstanceTypes(instancetypes.AWSSupportedInstanceFamilies), formatInstanceTypes(instancetypes.AzureCVMInstanceTypes), formatInstanceTypes(instancetypes.AzureTrustedLaunchInstanceTypes), formatInstanceTypes(instancetypes.GCPInstanceTypes)) +STACKIT instance types: +%v +`, + formatInstanceTypes(instancetypes.AWSSNPSupportedInstanceFamilies), + formatInstanceTypes(instancetypes.AWSSupportedInstanceFamilies), + formatInstanceTypes(instancetypes.AzureTDXInstanceTypes), + formatInstanceTypes(instancetypes.AzureSNPInstanceTypes), + formatInstanceTypes(instancetypes.AzureTrustedLaunchInstanceTypes), + formatInstanceTypes(instancetypes.GCPInstanceTypes), + formatInstanceTypes(instancetypes.STACKITInstanceTypes), + ) } func formatInstanceTypes(types []string) string { diff --git a/cli/internal/cmd/configkubernetesversions.go b/cli/internal/cmd/configkubernetesversions.go index 54183039d..4cc24da6a 100644 --- a/cli/internal/cmd/configkubernetesversions.go +++ b/cli/internal/cmd/configkubernetesversions.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/configmigrate.go b/cli/internal/cmd/configmigrate.go index f8bf7190a..2534ae3df 100644 --- a/cli/internal/cmd/configmigrate.go +++ b/cli/internal/cmd/configmigrate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/create.go b/cli/internal/cmd/create.go index 994c7e840..824ea1a2a 100644 --- a/cli/internal/cmd/create.go +++ b/cli/internal/cmd/create.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/create_test.go b/cli/internal/cmd/create_test.go index fe3119ee3..126ec80f8 100644 --- a/cli/internal/cmd/create_test.go +++ b/cli/internal/cmd/create_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -122,7 +122,7 @@ func TestCreate(t *testing.T) { wantErr: true, }, "config does not exist": { - setupFs: func(a *require.Assertions, p cloudprovider.Provider) afero.Fs { return afero.NewMemMapFs() }, + setupFs: func(_ *require.Assertions, _ cloudprovider.Provider) afero.Fs { return afero.NewMemMapFs() }, creator: &stubCloudCreator{ state: infraState, planDiff: true, diff --git a/cli/internal/cmd/iam.go b/cli/internal/cmd/iam.go index 229a0b2ee..1208ad84f 100644 --- a/cli/internal/cmd/iam.go +++ b/cli/internal/cmd/iam.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/iamcreate.go b/cli/internal/cmd/iamcreate.go index 337bf5f86..85f18b91d 100644 --- a/cli/internal/cmd/iamcreate.go +++ b/cli/internal/cmd/iamcreate.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -29,6 +29,9 @@ var ( regionRegex = regexp.MustCompile(`^\w+-\w+[0-9]$`) // Source: https://cloud.google.com/resource-manager/reference/rest/v1/projects. gcpIDRegex = regexp.MustCompile(`^[a-z][-a-z0-9]{4,28}[a-z0-9]$`) + + // We currently append 6 characters to the prefix, therefore we remove 6 characters from the gcpIDRegex. + gcpPrefixRegex = regexp.MustCompile(`^[a-z][-a-z0-9]{4,22}[a-z0-9]$`) ) // newIAMCreateCmd returns a new cobra.Command for the iam create parent command. It needs another verb, and does nothing on its own. @@ -82,7 +85,6 @@ func runIAMCreate(cmd *cobra.Command, providerCreator providerIAMCreator, provid if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() iamCreator := &iamCreator{ cmd: cmd, @@ -134,7 +136,7 @@ func (c *iamCreator) create(ctx context.Context) error { var conf config.Config if c.flags.updateConfig { - c.log.Debugf("Parsing config %s", c.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) + c.log.Debug(fmt.Sprintf("Parsing config %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) if err := c.fileHandler.ReadYAML(constants.ConfigFilename, &conf); err != nil { return fmt.Errorf("error reading the configuration file: %w", err) } @@ -154,7 +156,7 @@ func (c *iamCreator) create(ctx context.Context) error { return err } c.cmd.Println() // Print empty line to separate after spinner ended. - c.log.Debugf("Successfully created the IAM cloud resources") + c.log.Debug("Successfully created the IAM cloud resources") err = c.providerCreator.parseAndWriteIDFile(iamFile, c.fileHandler) if err != nil { @@ -162,7 +164,7 @@ func (c *iamCreator) create(ctx context.Context) error { } if c.flags.updateConfig { - c.log.Debugf("Writing IAM configuration to %s", c.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) + c.log.Debug(fmt.Sprintf("Writing IAM configuration to %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) c.providerCreator.writeOutputValuesToConfig(&conf, iamFile) if err := c.fileHandler.WriteYAML(constants.ConfigFilename, conf, file.OptOverwrite); err != nil { return err diff --git a/cli/internal/cmd/iamcreate_test.go b/cli/internal/cmd/iamcreate_test.go index 19aba50db..427a0a262 100644 --- a/cli/internal/cmd/iamcreate_test.go +++ b/cli/internal/cmd/iamcreate_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -67,7 +67,7 @@ func TestParseIDFile(t *testing.T) { func TestIAMCreateAWS(t *testing.T) { defaultFs := createFSWithConfig(*createConfig(cloudprovider.AWS)) - readOnlyFs := func(require *require.Assertions, provider cloudprovider.Provider, existingConfigFiles []string, existingDirs []string) afero.Fs { + readOnlyFs := func(_ *require.Assertions, _ cloudprovider.Provider, _ []string, _ []string) afero.Fs { fs := afero.NewReadOnlyFs(afero.NewMemMapFs()) return fs } @@ -257,7 +257,7 @@ func TestIAMCreateAWS(t *testing.T) { func TestIAMCreateAzure(t *testing.T) { defaultFs := createFSWithConfig(*createConfig(cloudprovider.Azure)) - readOnlyFs := func(require *require.Assertions, provider cloudprovider.Provider, existingConfigFiles []string, existingDirs []string) afero.Fs { + readOnlyFs := func(_ *require.Assertions, _ cloudprovider.Provider, _ []string, _ []string) afero.Fs { fs := afero.NewReadOnlyFs(afero.NewMemMapFs()) return fs } @@ -434,7 +434,7 @@ func TestIAMCreateAzure(t *testing.T) { func TestIAMCreateGCP(t *testing.T) { defaultFs := createFSWithConfig(*createConfig(cloudprovider.GCP)) - readOnlyFs := func(require *require.Assertions, provider cloudprovider.Provider, existingConfigFiles []string, existingDirs []string) afero.Fs { + readOnlyFs := func(_ *require.Assertions, _ cloudprovider.Provider, _ []string, _ []string) afero.Fs { fs := afero.NewReadOnlyFs(afero.NewMemMapFs()) return fs } @@ -456,6 +456,7 @@ func TestIAMCreateGCP(t *testing.T) { creator *stubIAMCreator zoneFlag string serviceAccountIDFlag string + namePrefixFlag string projectIDFlag string yesFlag bool updateConfigFlag bool @@ -466,6 +467,14 @@ func TestIAMCreateGCP(t *testing.T) { wantErr bool }{ "iam create gcp": { + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + yesFlag: true, + }, + "iam create gcp with deprecated serice account flag": { setupFs: defaultFs, creator: &stubIAMCreator{id: validIAMIDFile}, zoneFlag: "europe-west1-a", @@ -474,91 +483,91 @@ func TestIAMCreateGCP(t *testing.T) { yesFlag: true, }, "iam create gcp with existing config": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - yesFlag: true, - existingConfigFiles: []string{constants.ConfigFilename}, + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + yesFlag: true, + existingConfigFiles: []string{constants.ConfigFilename}, }, "iam create gcp --update-config": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - updateConfigFlag: true, - yesFlag: true, - existingConfigFiles: []string{constants.ConfigFilename}, + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + updateConfigFlag: true, + yesFlag: true, + existingConfigFiles: []string{constants.ConfigFilename}, }, "iam create gcp existing terraform dir": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", existingDirs: []string{constants.TerraformIAMWorkingDir}, yesFlag: true, wantErr: true, }, "iam create gcp invalid b64": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: invalidIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - yesFlag: true, - wantErr: true, + setupFs: defaultFs, + creator: &stubIAMCreator{id: invalidIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + yesFlag: true, + wantErr: true, }, "interactive": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - stdin: "yes\n", + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + stdin: "yes\n", }, "interactive update config": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - stdin: "yes\n", - updateConfigFlag: true, - existingConfigFiles: []string{constants.ConfigFilename}, + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + stdin: "yes\n", + updateConfigFlag: true, + existingConfigFiles: []string{constants.ConfigFilename}, }, "interactive abort": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - stdin: "no\n", - wantAbort: true, + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + stdin: "no\n", + wantAbort: true, }, "interactive abort update config": { - setupFs: defaultFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - stdin: "no\n", - wantAbort: true, - updateConfigFlag: true, - existingConfigFiles: []string{constants.ConfigFilename}, + setupFs: defaultFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + stdin: "no\n", + wantAbort: true, + updateConfigFlag: true, + existingConfigFiles: []string{constants.ConfigFilename}, }, "unwritable fs": { - setupFs: readOnlyFs, - creator: &stubIAMCreator{id: validIAMIDFile}, - zoneFlag: "europe-west1-a", - serviceAccountIDFlag: "constell-test", - projectIDFlag: "constell-1234", - yesFlag: true, - updateConfigFlag: true, - wantErr: true, + setupFs: readOnlyFs, + creator: &stubIAMCreator{id: validIAMIDFile}, + zoneFlag: "europe-west1-a", + namePrefixFlag: "constell-test", + projectIDFlag: "constell-1234", + yesFlag: true, + updateConfigFlag: true, + wantErr: true, }, } @@ -590,6 +599,7 @@ func TestIAMCreateGCP(t *testing.T) { flags: gcpIAMCreateFlags{ zone: tc.zoneFlag, serviceAccountID: tc.serviceAccountIDFlag, + namePrefix: tc.serviceAccountIDFlag, projectID: tc.projectIDFlag, }, }, @@ -684,7 +694,7 @@ func TestValidateConfigWithFlagCompatibility(t *testing.T) { } func createFSWithConfig(cfg config.Config) func(require *require.Assertions, provider cloudprovider.Provider, existingConfigFiles []string, existingDirs []string) afero.Fs { - return func(require *require.Assertions, provider cloudprovider.Provider, existingConfigFiles []string, existingDirs []string) afero.Fs { + return func(require *require.Assertions, _ cloudprovider.Provider, existingConfigFiles []string, existingDirs []string) afero.Fs { fs := afero.NewMemMapFs() fileHandler := file.NewHandler(fs) for _, f := range existingConfigFiles { diff --git a/cli/internal/cmd/iamcreateaws.go b/cli/internal/cmd/iamcreateaws.go index b648b87fc..1a29076c5 100644 --- a/cli/internal/cmd/iamcreateaws.go +++ b/cli/internal/cmd/iamcreateaws.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/iamcreateazure.go b/cli/internal/cmd/iamcreateazure.go index d80bcb654..f2ee7982b 100644 --- a/cli/internal/cmd/iamcreateazure.go +++ b/cli/internal/cmd/iamcreateazure.go @@ -1,12 +1,14 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd import ( + "errors" "fmt" + "os" "github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" @@ -26,6 +28,7 @@ func newIAMCreateAzureCmd() *cobra.Command { RunE: runIAMCreateAzure, } + cmd.Flags().String("subscriptionID", "", "subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set") cmd.Flags().String("resourceGroup", "", "name prefix of the two resource groups your cluster / IAM resources will be created in (required)") must(cobra.MarkFlagRequired(cmd.Flags(), "resourceGroup")) cmd.Flags().String("region", "", "region the resources will be created in, e.g., westus (required)") @@ -45,6 +48,7 @@ func runIAMCreateAzure(cmd *cobra.Command, _ []string) error { // azureIAMCreateFlags contains the parsed flags of the iam create azure command. type azureIAMCreateFlags struct { + subscriptionID string region string resourceGroup string servicePrincipal string @@ -52,6 +56,14 @@ type azureIAMCreateFlags struct { func (f *azureIAMCreateFlags) parse(flags *pflag.FlagSet) error { var err error + f.subscriptionID, err = flags.GetString("subscriptionID") + if err != nil { + return fmt.Errorf("getting 'subscriptionID' flag: %w", err) + } + if f.subscriptionID == "" && os.Getenv("ARM_SUBSCRIPTION_ID") == "" { + return errors.New("either flag 'subscriptionID' or environment variable 'ARM_SUBSCRIPTION_ID' must be set") + } + f.region, err = flags.GetString("region") if err != nil { return fmt.Errorf("getting 'region' flag: %w", err) @@ -75,6 +87,7 @@ type azureIAMCreator struct { func (c *azureIAMCreator) getIAMConfigOptions() *cloudcmd.IAMConfigOptions { return &cloudcmd.IAMConfigOptions{ Azure: cloudcmd.AzureIAMConfig{ + SubscriptionID: c.flags.subscriptionID, Location: c.flags.region, ResourceGroup: c.flags.resourceGroup, ServicePrincipal: c.flags.servicePrincipal, @@ -83,6 +96,7 @@ func (c *azureIAMCreator) getIAMConfigOptions() *cloudcmd.IAMConfigOptions { } func (c *azureIAMCreator) printConfirmValues(cmd *cobra.Command) { + cmd.Printf("Subscription ID:\t%s\n", c.flags.subscriptionID) cmd.Printf("Region:\t\t\t%s\n", c.flags.region) cmd.Printf("Resource Group:\t\t%s\n", c.flags.resourceGroup) cmd.Printf("Service Principal:\t%s\n\n", c.flags.servicePrincipal) diff --git a/cli/internal/cmd/iamcreategcp.go b/cli/internal/cmd/iamcreategcp.go index b6c55e5d1..f2c6c6ead 100644 --- a/cli/internal/cmd/iamcreategcp.go +++ b/cli/internal/cmd/iamcreategcp.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -31,13 +31,19 @@ func newIAMCreateGCPCmd() *cobra.Command { cmd.Flags().String("zone", "", "GCP zone the cluster will be deployed in (required)\n"+ "Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available") must(cobra.MarkFlagRequired(cmd.Flags(), "zone")) - cmd.Flags().String("serviceAccountID", "", "ID for the service account that will be created (required)\n"+ - "Must be 6 to 30 lowercase letters, digits, or hyphens.") - must(cobra.MarkFlagRequired(cmd.Flags(), "serviceAccountID")) + + cmd.Flags().String("serviceAccountID", "", "[Deprecated use \"--prefix\"]ID for the service account that will be created (required)\n"+ + "Must be 6 to 30 lowercase letters, digits, or hyphens. This flag is mutually exclusive with --prefix.") + cmd.Flags().String("prefix", "", "Prefix for the service account ID and VM ID that will be created (required)\n"+ + "Must be letters, digits, or hyphens.") + cmd.Flags().String("projectID", "", "ID of the GCP project the configuration will be created in (required)\n"+ "Find it on the welcome screen of your project: https://console.cloud.google.com/welcome") must(cobra.MarkFlagRequired(cmd.Flags(), "projectID")) + cmd.MarkFlagsMutuallyExclusive([]string{"prefix", "serviceAccountID"}...) + must(cmd.Flags().MarkDeprecated("serviceAccountID", "use --prefix instead")) + return cmd } @@ -53,6 +59,7 @@ func runIAMCreateGCP(cmd *cobra.Command, _ []string) error { type gcpIAMCreateFlags struct { rootFlags serviceAccountID string + namePrefix string zone string region string projectID string @@ -91,9 +98,18 @@ func (f *gcpIAMCreateFlags) parse(flags *pflag.FlagSet) error { if err != nil { return fmt.Errorf("getting 'serviceAccountID' flag: %w", err) } - if !gcpIDRegex.MatchString(f.serviceAccountID) { + if f.serviceAccountID != "" && !gcpIDRegex.MatchString(f.serviceAccountID) { return fmt.Errorf("serviceAccountID %q doesn't match %s", f.serviceAccountID, gcpIDRegex) } + + f.namePrefix, err = flags.GetString("prefix") + if err != nil { + return fmt.Errorf("getting 'prefix' flag: %w", err) + } + if f.namePrefix != "" && !gcpPrefixRegex.MatchString(f.namePrefix) { + return fmt.Errorf("prefix %q doesn't match %s", f.namePrefix, gcpIDRegex) + } + return nil } @@ -109,13 +125,19 @@ func (c *gcpIAMCreator) getIAMConfigOptions() *cloudcmd.IAMConfigOptions { Region: c.flags.region, ProjectID: c.flags.projectID, ServiceAccountID: c.flags.serviceAccountID, + NamePrefix: c.flags.namePrefix, }, } } func (c *gcpIAMCreator) printConfirmValues(cmd *cobra.Command) { cmd.Printf("Project ID:\t\t%s\n", c.flags.projectID) - cmd.Printf("Service Account ID:\t%s\n", c.flags.serviceAccountID) + if c.flags.namePrefix != "" { + cmd.Printf("Name Prefix:\t\t%s\n", c.flags.namePrefix) + } + if c.flags.serviceAccountID != "" { + cmd.Printf("Service Account ID:\t%s\n", c.flags.serviceAccountID) + } cmd.Printf("Region:\t\t\t%s\n", c.flags.region) cmd.Printf("Zone:\t\t\t%s\n\n", c.flags.zone) } @@ -127,11 +149,12 @@ func (c *gcpIAMCreator) printOutputValues(cmd *cobra.Command, _ cloudcmd.IAMOutp cmd.Printf("serviceAccountKeyPath:\t%s\n\n", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename)) } -func (c *gcpIAMCreator) writeOutputValuesToConfig(conf *config.Config, _ cloudcmd.IAMOutput) { +func (c *gcpIAMCreator) writeOutputValuesToConfig(conf *config.Config, out cloudcmd.IAMOutput) { conf.Provider.GCP.Project = c.flags.projectID conf.Provider.GCP.ServiceAccountKeyPath = constants.GCPServiceAccountKeyFilename // File was created in workspace, so only the filename is needed. conf.Provider.GCP.Region = c.flags.region conf.Provider.GCP.Zone = c.flags.zone + conf.Provider.GCP.IAMServiceAccountVM = out.GCPOutput.IAMServiceAccountVM for groupName, group := range conf.NodeGroups { group.Zone = c.flags.zone conf.NodeGroups[groupName] = group diff --git a/cli/internal/cmd/iamdestroy.go b/cli/internal/cmd/iamdestroy.go index 667218b81..abcf9290f 100644 --- a/cli/internal/cmd/iamdestroy.go +++ b/cli/internal/cmd/iamdestroy.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -58,7 +58,6 @@ func runIAMDestroy(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() spinner := newSpinner(cmd.ErrOrStderr()) destroyer := cloudcmd.NewIAMDestroyer() fsHandler := file.NewHandler(afero.NewOsFs()) @@ -78,25 +77,25 @@ type destroyCmd struct { func (c *destroyCmd) iamDestroy(cmd *cobra.Command, spinner spinnerInterf, destroyer iamDestroyer, fsHandler file.Handler) error { // check if there is a possibility that the cluster is still running by looking out for specific files - c.log.Debugf("Checking if %q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) + c.log.Debug(fmt.Sprintf("Checking if %q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename))) if _, err := fsHandler.Stat(constants.AdminConfFilename); !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("file %q still exists, please make sure to terminate your cluster before destroying your IAM configuration", c.flags.pathPrefixer.PrefixPrintablePath(constants.AdminConfFilename)) } - c.log.Debugf("Checking if %q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename)) + c.log.Debug(fmt.Sprintf("Checking if %q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename))) if _, err := fsHandler.Stat(constants.StateFilename); !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("file %q still exists, please make sure to terminate your cluster before destroying your IAM configuration", c.flags.pathPrefixer.PrefixPrintablePath(constants.StateFilename)) } gcpFileExists := false - c.log.Debugf("Checking if %q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename)) + c.log.Debug(fmt.Sprintf("Checking if %q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename))) if _, err := fsHandler.Stat(constants.GCPServiceAccountKeyFilename); err != nil { if !errors.Is(err, os.ErrNotExist) { return err } } else { - c.log.Debugf("%q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename)) + c.log.Debug(fmt.Sprintf("%q exists", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename))) gcpFileExists = true } @@ -117,7 +116,7 @@ func (c *destroyCmd) iamDestroy(cmd *cobra.Command, spinner spinnerInterf, destr } if gcpFileExists { - c.log.Debugf("Starting to delete %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename)) + c.log.Debug(fmt.Sprintf("Starting to delete %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename))) proceed, err := c.deleteGCPServiceAccountKeyFile(cmd, destroyer, fsHandler) if err != nil { return err @@ -128,7 +127,7 @@ func (c *destroyCmd) iamDestroy(cmd *cobra.Command, spinner spinnerInterf, destr } } - c.log.Debugf("Starting to destroy IAM configuration") + c.log.Debug("Starting to destroy IAM configuration") spinner.Start("Destroying IAM configuration", false) defer spinner.Stop() @@ -144,18 +143,18 @@ func (c *destroyCmd) iamDestroy(cmd *cobra.Command, spinner spinnerInterf, destr func (c *destroyCmd) deleteGCPServiceAccountKeyFile(cmd *cobra.Command, destroyer iamDestroyer, fsHandler file.Handler) (bool, error) { var fileSaKey gcpshared.ServiceAccountKey - c.log.Debugf("Parsing %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename)) + c.log.Debug(fmt.Sprintf("Parsing %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename))) if err := fsHandler.ReadJSON(constants.GCPServiceAccountKeyFilename, &fileSaKey); err != nil { return false, err } - c.log.Debugf("Getting service account key from the tfstate") + c.log.Debug("Getting service account key from the tfstate") tfSaKey, err := destroyer.GetTfStateServiceAccountKey(cmd.Context(), constants.TerraformIAMWorkingDir) if err != nil { return false, err } - c.log.Debugf("Checking if keys are the same") + c.log.Debug("Checking if keys are the same") if tfSaKey != fileSaKey { cmd.Printf( "The key in %q don't match up with your Terraform state. %q will not be deleted.\n", @@ -169,6 +168,6 @@ func (c *destroyCmd) deleteGCPServiceAccountKeyFile(cmd *cobra.Command, destroye return false, err } - c.log.Debugf("Successfully deleted %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename)) + c.log.Debug(fmt.Sprintf("Successfully deleted %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.GCPServiceAccountKeyFilename))) return true, nil } diff --git a/cli/internal/cmd/iamdestroy_test.go b/cli/internal/cmd/iamdestroy_test.go index e6dd4feb2..675f8df5a 100644 --- a/cli/internal/cmd/iamdestroy_test.go +++ b/cli/internal/cmd/iamdestroy_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/iamupgradeapply.go b/cli/internal/cmd/iamupgradeapply.go index 7f1e98544..bf8f7b275 100644 --- a/cli/internal/cmd/iamupgradeapply.go +++ b/cli/internal/cmd/iamupgradeapply.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -149,7 +149,7 @@ func (i iamUpgradeApplyCmd) iamUpgradeApply(cmd *cobra.Command, iamUpgrader iamU return errors.New("IAM upgrade aborted by user") } } - i.log.Debugf("Applying Terraform IAM migrations") + i.log.Debug("Applying Terraform IAM migrations") if err := iamUpgrader.ApplyIAMUpgrade(cmd.Context(), conf.GetProvider()); err != nil { return fmt.Errorf("applying terraform migrations: %w", err) } diff --git a/cli/internal/cmd/iamupgradeapply_test.go b/cli/internal/cmd/iamupgradeapply_test.go index e1d4c19ce..32074f56c 100644 --- a/cli/internal/cmd/iamupgradeapply_test.go +++ b/cli/internal/cmd/iamupgradeapply_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -171,14 +171,6 @@ type stubConfigFetcher struct { fetchLatestErr error } -func (s *stubConfigFetcher) FetchSEVSNPVersion(context.Context, attestationconfigapi.SEVSNPVersionAPI) (attestationconfigapi.SEVSNPVersionAPI, error) { - panic("not implemented") -} - -func (s *stubConfigFetcher) FetchSEVSNPVersionList(context.Context, attestationconfigapi.SEVSNPVersionList) (attestationconfigapi.SEVSNPVersionList, error) { - panic("not implemented") -} - -func (s *stubConfigFetcher) FetchSEVSNPVersionLatest(context.Context, variant.Variant) (attestationconfigapi.SEVSNPVersionAPI, error) { - return attestationconfigapi.SEVSNPVersionAPI{}, s.fetchLatestErr +func (s *stubConfigFetcher) FetchLatestVersion(context.Context, variant.Variant) (attestationconfigapi.Entry, error) { + return attestationconfigapi.Entry{}, s.fetchLatestErr } diff --git a/cli/internal/cmd/init.go b/cli/internal/cmd/init.go index 166195d49..ce10f67e3 100644 --- a/cli/internal/cmd/init.go +++ b/cli/internal/cmd/init.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd import ( - "context" "fmt" "io" "os" @@ -72,7 +71,7 @@ func (c *kubeconfigMerger) mergeConfigs(configPath string, fileHandler file.Hand clientcmd.RecommendedHomeFile, configPath, // our config should overwrite the default config } - c.log.Debugf("Kubeconfig file loading precedence: %v", loadingRules.Precedence) + c.log.Debug(fmt.Sprintf("Kubeconfig file loading precedence: %v", loadingRules.Precedence)) // merge the kubeconfigs cfg, err := loadingRules.Load() @@ -82,7 +81,7 @@ func (c *kubeconfigMerger) mergeConfigs(configPath string, fileHandler file.Hand // Set the current context to the cluster we just created cfg.CurrentContext = constellConfig.CurrentContext - c.log.Debugf("Set current context to %s", cfg.CurrentContext) + c.log.Debug(fmt.Sprintf("Set current context to %q", cfg.CurrentContext)) json, err := runtime.Encode(clientcodec.Codec, cfg) if err != nil { @@ -97,7 +96,7 @@ func (c *kubeconfigMerger) mergeConfigs(configPath string, fileHandler file.Hand if err := fileHandler.Write(clientcmd.RecommendedHomeFile, mergedKubeconfig, file.OptOverwrite); err != nil { return fmt.Errorf("writing merged kubeconfig to file: %w", err) } - c.log.Debugf("Merged kubeconfig into default config file: %s", clientcmd.RecommendedHomeFile) + c.log.Debug(fmt.Sprintf("Merged kubeconfig into default config file: %q", clientcmd.RecommendedHomeFile)) return nil } @@ -106,5 +105,5 @@ func (c *kubeconfigMerger) kubeconfigEnvVar() string { } type grpcDialer interface { - Dial(ctx context.Context, target string) (*grpc.ClientConn, error) + Dial(target string) (*grpc.ClientConn, error) } diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index 169949d1c..63ee594fe 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -217,7 +217,7 @@ func TestInitialize(t *testing.T) { require.NoError(fileHandler.WriteJSON(serviceAccPath, tc.serviceAccKey, file.OptNone)) } - ctx := context.Background() + ctx := t.Context() ctx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() cmd.SetContext(ctx) @@ -278,8 +278,16 @@ type stubHelmApplier struct { err error } +func (s stubHelmApplier) AnnotateCoreDNSResources(_ context.Context) error { + return nil +} + +func (s stubHelmApplier) CleanupCoreDNSResources(_ context.Context) error { + return nil +} + func (s stubHelmApplier) PrepareHelmCharts( - _ helm.Options, _ *state.State, _ string, _ uri.MasterSecret, _ *config.OpenStackConfig, + _ helm.Options, _ *state.State, _ string, _ uri.MasterSecret, ) (helm.Applier, bool, error) { return stubRunner{}, false, s.err } @@ -369,7 +377,7 @@ func TestWriteOutput(t *testing.T) { spinner: &nopSpinner{}, merger: &stubMerger{}, log: logger.NewTest(t), - applier: constellation.NewApplier(logger.NewTest(t), &nopSpinner{}, nil), + applier: constellation.NewApplier(logger.NewTest(t), &nopSpinner{}, constellation.ApplyContextCLI, nil), } err = i.writeInitOutput(stateFile, initOutput, false, &out, measurementSalt) require.NoError(err) @@ -438,12 +446,12 @@ func TestGenerateMasterSecret(t *testing.T) { wantErr: true, }, "file does not exist": { - createFileFunc: func(handler file.Handler) error { return nil }, + createFileFunc: func(_ file.Handler) error { return nil }, fs: afero.NewMemMapFs, wantErr: false, }, "file not writeable": { - createFileFunc: func(handler file.Handler) error { return nil }, + createFileFunc: func(_ file.Handler) error { return nil }, fs: func() afero.Fs { return afero.NewReadOnlyFs(afero.NewMemMapFs()) }, wantErr: true, }, @@ -461,7 +469,7 @@ func TestGenerateMasterSecret(t *testing.T) { i := &applyCmd{ fileHandler: fileHandler, log: logger.NewTest(t), - applier: constellation.NewApplier(logger.NewTest(t), &nopSpinner{}, nil), + applier: constellation.NewApplier(logger.NewTest(t), &nopSpinner{}, constellation.ApplyContextCLI, nil), } secret, err := i.generateAndPersistMasterSecret(&out) @@ -531,9 +539,10 @@ func defaultConfigWithExpectedMeasurements(t *testing.T, conf *config.Config, cs conf.Provider.GCP.Project = "test-project" conf.Provider.GCP.Zone = "test-zone" conf.Provider.GCP.ServiceAccountKeyPath = "test-key-path" - conf.Attestation.GCPSEVES.Measurements[4] = measurements.WithAllBytes(0x44, measurements.Enforce, measurements.PCRMeasurementLength) - conf.Attestation.GCPSEVES.Measurements[9] = measurements.WithAllBytes(0x11, measurements.Enforce, measurements.PCRMeasurementLength) - conf.Attestation.GCPSEVES.Measurements[12] = measurements.WithAllBytes(0xcc, measurements.Enforce, measurements.PCRMeasurementLength) + conf.Provider.GCP.IAMServiceAccountVM = "example@example.com" + conf.Attestation.GCPSEVSNP.Measurements[4] = measurements.WithAllBytes(0x44, measurements.Enforce, measurements.PCRMeasurementLength) + conf.Attestation.GCPSEVSNP.Measurements[9] = measurements.WithAllBytes(0x11, measurements.Enforce, measurements.PCRMeasurementLength) + conf.Attestation.GCPSEVSNP.Measurements[12] = measurements.WithAllBytes(0xcc, measurements.Enforce, measurements.PCRMeasurementLength) zone = "europe-west3-b" instanceType = "n2d-standard-4" diskType = "pd-ssd" diff --git a/cli/internal/cmd/license_enterprise.go b/cli/internal/cmd/license_enterprise.go index 2bc1d8797..399de4524 100644 --- a/cli/internal/cmd/license_enterprise.go +++ b/cli/internal/cmd/license_enterprise.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -22,18 +22,22 @@ import ( // with the license server. If no license file is present or if errors // occur during the check, the user is informed and the community license // is used. It is a no-op in the open source version of Constellation. -func (a *applyCmd) checkLicenseFile(cmd *cobra.Command, csp cloudprovider.Provider) { +func (a *applyCmd) checkLicenseFile(cmd *cobra.Command, csp cloudprovider.Provider, useMarketplaceImage bool) { var licenseID string - a.log.Debugf("Running license check") + a.log.Debug("Running license check") readBytes, err := a.fileHandler.Read(constants.LicenseFilename) - if errors.Is(err, fs.ErrNotExist) { - cmd.Printf("Using community license.\n") + switch { + case useMarketplaceImage: + cmd.Println("Using marketplace image billing.") + licenseID = license.MarketplaceLicense + case errors.Is(err, fs.ErrNotExist): + cmd.Println("Using community license.") licenseID = license.CommunityLicense - } else if err != nil { + case err != nil: cmd.Printf("Error: %v\nContinuing with community license.\n", err) licenseID = license.CommunityLicense - } else { + default: cmd.Printf("Constellation license found!\n") licenseID, err = license.FromBytes(readBytes) if err != nil { @@ -42,15 +46,17 @@ func (a *applyCmd) checkLicenseFile(cmd *cobra.Command, csp cloudprovider.Provid } } - quota, err := a.applier.CheckLicense(cmd.Context(), csp, licenseID) - if err != nil { + quota, err := a.applier.CheckLicense(cmd.Context(), csp, !a.flags.skipPhases.contains(skipInitPhase), licenseID) + if err != nil && !useMarketplaceImage { cmd.Printf("Unable to contact license server.\n") cmd.Printf("Please keep your vCPU quota in mind.\n") + } else if licenseID == license.MarketplaceLicense { + // Do nothing. Billing is handled by the marketplace. } else if licenseID == license.CommunityLicense { cmd.Printf("For details, see https://docs.edgeless.systems/constellation/overview/license\n") } else { cmd.Printf("Please keep your vCPU quota (%d) in mind.\n", quota) } - a.log.Debugf("Checked license") + a.log.Debug("Checked license") } diff --git a/cli/internal/cmd/license_oss.go b/cli/internal/cmd/license_oss.go index 8fba56114..7d584a78c 100644 --- a/cli/internal/cmd/license_oss.go +++ b/cli/internal/cmd/license_oss.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -17,4 +17,4 @@ import ( // with the license server. If no license file is present or if errors // occur during the check, the user is informed and the community license // is used. It is a no-op in the open source version of Constellation. -func (a *applyCmd) checkLicenseFile(*cobra.Command, cloudprovider.Provider) {} +func (a *applyCmd) checkLicenseFile(*cobra.Command, cloudprovider.Provider, bool) {} diff --git a/cli/internal/cmd/log.go b/cli/internal/cmd/log.go index 463e1f5b6..d112fcc37 100644 --- a/cli/internal/cmd/log.go +++ b/cli/internal/cmd/log.go @@ -1,31 +1,31 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd import ( + "log/slog" + "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) type debugLog interface { - Debugf(format string, args ...any) - Sync() + Debug(msg string, args ...any) } func newCLILogger(cmd *cobra.Command) (debugLog, error) { - logLvl := zapcore.InfoLevel + logLvl := slog.LevelInfo debugLog, err := cmd.Flags().GetBool("debug") if err != nil { return nil, err } if debugLog { - logLvl = zapcore.DebugLevel + logLvl = slog.LevelDebug } - return logger.New(logger.PlainLog, logLvl), nil + return logger.NewTextLogger(logLvl), nil } diff --git a/cli/internal/cmd/maapatch.go b/cli/internal/cmd/maapatch.go index 7db1abb5c..11e86051a 100644 --- a/cli/internal/cmd/maapatch.go +++ b/cli/internal/cmd/maapatch.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -23,7 +23,7 @@ func NewMaaPatchCmd() *cobra.Command { Long: "Patch the MAA's attestation policy.", Args: cobra.MatchAll( cobra.ExactArgs(1), - func(cmd *cobra.Command, args []string) error { + func(_ *cobra.Command, args []string) error { if _, err := url.Parse(args[0]); err != nil { return fmt.Errorf("argument %s is not a valid URL: %w", args[0], err) } @@ -47,7 +47,6 @@ func runPatchMAA(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() p := maa.NewAzurePolicyPatcher() @@ -57,7 +56,7 @@ func runPatchMAA(cmd *cobra.Command, args []string) error { } func (c *maaPatchCmd) patchMAA(cmd *cobra.Command, attestationURL string) error { - c.log.Debugf("Using attestation URL %s", attestationURL) + c.log.Debug(fmt.Sprintf("Using attestation URL %q", attestationURL)) if err := c.patcher.Patch(cmd.Context(), attestationURL); err != nil { return fmt.Errorf("patching MAA attestation policy: %w", err) diff --git a/cli/internal/cmd/maapatch_test.go b/cli/internal/cmd/maapatch_test.go index bbd0e165f..bca2d0ee7 100644 --- a/cli/internal/cmd/maapatch_test.go +++ b/cli/internal/cmd/maapatch_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/mini.go b/cli/internal/cmd/mini.go index 4b4774f67..7d6fe087d 100644 --- a/cli/internal/cmd/mini.go +++ b/cli/internal/cmd/mini.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/minidown.go b/cli/internal/cmd/minidown.go index 594312e28..525b8539f 100644 --- a/cli/internal/cmd/minidown.go +++ b/cli/internal/cmd/minidown.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/miniup.go b/cli/internal/cmd/miniup.go index ffe254e90..1b8c0984a 100644 --- a/cli/internal/cmd/miniup.go +++ b/cli/internal/cmd/miniup.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -50,7 +50,6 @@ func runUp(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() m := &miniUpCmd{ log: log, @@ -152,7 +151,7 @@ func (m *miniUpCmd) prepareConfig(cmd *cobra.Command) (*config.Config, error) { if err != nil { return nil, fmt.Errorf("mini default config is invalid: %v", err) } - m.log.Debugf("Prepared configuration") + m.log.Debug("Prepared configuration") return config, m.fileHandler.WriteYAML(constants.ConfigFilename, config, file.OptOverwrite) } diff --git a/cli/internal/cmd/miniup_cross.go b/cli/internal/cmd/miniup_cross.go index 4668cc8f3..2abd7c3b3 100644 --- a/cli/internal/cmd/miniup_cross.go +++ b/cli/internal/cmd/miniup_cross.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/miniup_linux_amd64.go b/cli/internal/cmd/miniup_linux_amd64.go index a28fe1486..c2c5a052e 100644 --- a/cli/internal/cmd/miniup_linux_amd64.go +++ b/cli/internal/cmd/miniup_linux_amd64.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -32,12 +32,12 @@ func (m *miniUpCmd) checkSystemRequirements(out io.Writer) error { return fmt.Errorf("creation of a QEMU based Constellation is not supported for %s/%s, a linux/amd64 platform is required", runtime.GOOS, runtime.GOARCH) } - m.log.Debugf("Checked arch and os") + m.log.Debug("Checked arch and os") // check if /dev/kvm exists if _, err := os.Stat("/dev/kvm"); err != nil { return fmt.Errorf("unable to access KVM device: %w", err) } - m.log.Debugf("Checked that /dev/kvm exists") + m.log.Debug("Checked that /dev/kvm exists") // check CPU cores if runtime.NumCPU() < 4 { return fmt.Errorf("insufficient CPU cores: %d, at least 4 cores are required by MiniConstellation", runtime.NumCPU()) @@ -45,7 +45,7 @@ func (m *miniUpCmd) checkSystemRequirements(out io.Writer) error { if runtime.NumCPU() < 6 { fmt.Fprintf(out, "WARNING: Only %d CPU cores available. This may cause performance issues.\n", runtime.NumCPU()) } - m.log.Debugf("Checked CPU cores - there are %d", runtime.NumCPU()) + m.log.Debug(fmt.Sprintf("Checked CPU cores - there are %d", runtime.NumCPU())) // check memory f, err := os.Open("/proc/meminfo") @@ -63,7 +63,7 @@ func (m *miniUpCmd) checkSystemRequirements(out io.Writer) error { } } } - m.log.Debugf("Scanned for available memory") + m.log.Debug("Scanned for available memory") memGB := memKB / 1024 / 1024 if memGB < 4 { return fmt.Errorf("insufficient memory: %dGB, at least 4GB of memory are required by MiniConstellation", memGB) @@ -71,7 +71,7 @@ func (m *miniUpCmd) checkSystemRequirements(out io.Writer) error { if memGB < 6 { fmt.Fprintln(out, "WARNING: Less than 6GB of memory available. This may cause performance issues.") } - m.log.Debugf("Checked available memory, you have %dGB available", memGB) + m.log.Debug(fmt.Sprintf("Checked available memory, you have %dGB available", memGB)) var stat unix.Statfs_t if err := unix.Statfs(".", &stat); err != nil { @@ -81,7 +81,7 @@ func (m *miniUpCmd) checkSystemRequirements(out io.Writer) error { if freeSpaceGB < 20 { return fmt.Errorf("insufficient disk space: %dGB, at least 20GB of disk space are required by MiniConstellation", freeSpaceGB) } - m.log.Debugf("Checked for free space available, you have %dGB available", freeSpaceGB) + m.log.Debug(fmt.Sprintf("Checked for free space available, you have %dGB available", freeSpaceGB)) return nil } diff --git a/cli/internal/cmd/pathprefix/pathprefix.go b/cli/internal/cmd/pathprefix/pathprefix.go index a70ccfee3..823406232 100644 --- a/cli/internal/cmd/pathprefix/pathprefix.go +++ b/cli/internal/cmd/pathprefix/pathprefix.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/cli/internal/cmd/recover.go b/cli/internal/cmd/recover.go index 3a490bc89..ab5d3ef14 100644 --- a/cli/internal/cmd/recover.go +++ b/cli/internal/cmd/recover.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -11,7 +11,6 @@ import ( "errors" "fmt" "io" - "net" "sync" "time" @@ -76,16 +75,15 @@ func runRecover(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() fileHandler := file.NewHandler(afero.NewOsFs()) newDialer := func(validator atls.Validator) *dialer.Dialer { - return dialer.New(nil, validator, &net.Dialer{}) + return dialer.New(nil, validator, nil) } r := &recoverCmd{log: log, configFetcher: attestationconfigapi.NewFetcher()} if err := r.flags.parse(cmd.Flags()); err != nil { return err } - r.log.Debugf("Using flags: %+v", r.flags) + r.log.Debug("Using flags", "debug", r.flags.debug, "endpoint", r.flags.endpoint, "force", r.flags.force) return r.recover(cmd, fileHandler, 5*time.Second, &recoverDoer{log: r.log}, newDialer) } @@ -94,12 +92,12 @@ func (r *recoverCmd) recover( doer recoverDoerInterface, newDialer func(validator atls.Validator) *dialer.Dialer, ) error { var masterSecret uri.MasterSecret - r.log.Debugf("Loading master secret file from %s", r.flags.pathPrefixer.PrefixPrintablePath(constants.MasterSecretFilename)) + r.log.Debug(fmt.Sprintf("Loading master secret file from %q", r.flags.pathPrefixer.PrefixPrintablePath(constants.MasterSecretFilename))) if err := fileHandler.ReadJSON(constants.MasterSecretFilename, &masterSecret); err != nil { return err } - r.log.Debugf("Loading configuration file from %q", r.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) + r.log.Debug(fmt.Sprintf("Loading configuration file from %q", r.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) conf, err := config.New(fileHandler, constants.ConfigFilename, r.configFetcher, r.flags.force) var configValidationErr *config.ValidationError if errors.As(err, &configValidationErr) { @@ -109,9 +107,8 @@ func (r *recoverCmd) recover( return err } - provider := conf.GetProvider() - r.log.Debugf("Got provider %s", provider.String()) - if provider == cloudprovider.Azure { + r.log.Debug(fmt.Sprintf("Got provider %q", conf.GetProvider())) + if conf.GetProvider() == cloudprovider.Azure { interval = 20 * time.Second // Azure LB takes a while to remove unhealthy instances } @@ -119,7 +116,7 @@ func (r *recoverCmd) recover( if err != nil { return fmt.Errorf("reading state file: %w", err) } - if err := stateFile.Validate(state.PostInit, provider); err != nil { + if err := stateFile.Validate(state.PostInit, conf.GetAttestationConfig().GetVariant()); err != nil { return fmt.Errorf("validating state file: %w", err) } @@ -131,16 +128,16 @@ func (r *recoverCmd) recover( conf.UpdateMAAURL(stateFile.Infrastructure.Azure.AttestationURL) } - r.log.Debugf("Creating aTLS Validator for %s", conf.GetAttestationConfig().GetVariant()) + r.log.Debug(fmt.Sprintf("Creating aTLS Validator for %q", conf.GetAttestationConfig().GetVariant())) validator, err := choose.Validator(conf.GetAttestationConfig(), warnLogger{cmd: cmd, log: r.log}) if err != nil { return fmt.Errorf("creating new validator: %w", err) } - r.log.Debugf("Created a new validator") + r.log.Debug("Created a new validator") doer.setDialer(newDialer(validator), endpoint) - r.log.Debugf("Set dialer for endpoint %s", endpoint) + r.log.Debug(fmt.Sprintf("Set dialer for endpoint %q", endpoint)) doer.setURIs(masterSecret.EncodeToURI(), uri.NoStoreURI) - r.log.Debugf("Set secrets") + r.log.Debug("Set secrets") if err := r.recoverCall(cmd.Context(), cmd.OutOrStdout(), interval, doer); err != nil { if grpcRetry.ServiceIsUnavailable(err) { return nil @@ -168,12 +165,12 @@ func (r *recoverCmd) recoverCall(ctx context.Context, out io.Writer, interval ti }) } - r.log.Debugf("Encountered error (retriable: %t): %s", retry, err) + r.log.Debug(fmt.Sprintf("Encountered error (retriable: %t): %q", retry, err)) return retry } retrier := retry.NewIntervalRetrier(doer, interval, retryOnceOnFailure) - r.log.Debugf("Created new interval retrier") + r.log.Debug("Created new interval retrier") err = retrier.Do(ctx) if err != nil { break @@ -181,7 +178,7 @@ func (r *recoverCmd) recoverCall(ctx context.Context, out io.Writer, interval ti fmt.Fprintln(out, "Pushed recovery key.") ctr++ } - r.log.Debugf("Retry counter is %d", ctr) + r.log.Debug(fmt.Sprintf("Retry counter is %d", ctr)) if ctr > 0 { fmt.Fprintf(out, "Recovered %d control-plane nodes.\n", ctr) } else if grpcRetry.ServiceIsUnavailable(err) { @@ -219,15 +216,15 @@ type recoverDoer struct { // Do performs the recover streaming rpc. func (d *recoverDoer) Do(ctx context.Context) (retErr error) { - conn, err := d.dialer.Dial(ctx, d.endpoint) + conn, err := d.dialer.Dial(d.endpoint) if err != nil { return fmt.Errorf("dialing recovery server: %w", err) } - d.log.Debugf("Dialed recovery server") + d.log.Debug("Dialed recovery server") defer conn.Close() protoClient := recoverproto.NewAPIClient(conn) - d.log.Debugf("Created protoClient") + d.log.Debug("Created protoClient") req := &recoverproto.RecoverMessage{ KmsUri: d.kmsURI, @@ -239,7 +236,7 @@ func (d *recoverDoer) Do(ctx context.Context) (retErr error) { return fmt.Errorf("calling recover: %w", err) } - d.log.Debugf("Received confirmation") + d.log.Debug("Received confirmation") return nil } diff --git a/cli/internal/cmd/recover_test.go b/cli/internal/cmd/recover_test.go index 41ca89817..af0817597 100644 --- a/cli/internal/cmd/recover_test.go +++ b/cli/internal/cmd/recover_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -138,7 +138,7 @@ func TestRecover(t *testing.T) { require := require.New(t) cmd := NewRecoverCmd() - cmd.SetContext(context.Background()) + cmd.SetContext(t.Context()) out := &bytes.Buffer{} cmd.SetOut(out) cmd.SetErr(out) @@ -225,7 +225,7 @@ func TestDoRecovery(t *testing.T) { log: r.log, } - err := recoverDoer.Do(context.Background()) + err := recoverDoer.Do(t.Context()) if tc.wantErr { assert.Error(err) } else { diff --git a/cli/internal/cmd/spinner.go b/cli/internal/cmd/spinner.go index 4184291a8..f979459c3 100644 --- a/cli/internal/cmd/spinner.go +++ b/cli/internal/cmd/spinner.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/spinner_test.go b/cli/internal/cmd/spinner_test.go index fab3ffa87..4cbf40d34 100644 --- a/cli/internal/cmd/spinner_test.go +++ b/cli/internal/cmd/spinner_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/ssh.go b/cli/internal/cmd/ssh.go new file mode 100644 index 000000000..84f9ed67a --- /dev/null +++ b/cli/internal/cmd/ssh.go @@ -0,0 +1,118 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package cmd + +import ( + "crypto/ed25519" + "crypto/rand" + "fmt" + "time" + + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/crypto" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/kms/setup" + "github.com/edgelesssys/constellation/v2/internal/kms/uri" + "github.com/spf13/afero" + "github.com/spf13/cobra" + + "golang.org/x/crypto/ssh" +) + +// NewSSHCmd returns a new cobra.Command for the ssh command. +func NewSSHCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "ssh", + Short: "Generate a certificate for emergency SSH access", + Long: "Generate a certificate for emergency SSH access to your SSH-enabled constellation cluster.", + Args: cobra.ExactArgs(0), + RunE: runSSH, + } + cmd.Flags().String("key", "", "the path to an existing SSH public key") + must(cmd.MarkFlagRequired("key")) + return cmd +} + +func runSSH(cmd *cobra.Command, _ []string) error { + fh := file.NewHandler(afero.NewOsFs()) + debugLogger, err := newDebugFileLogger(cmd, fh) + if err != nil { + return err + } + + keyPath, err := cmd.Flags().GetString("key") + if err != nil { + return fmt.Errorf("retrieving path to public key from flags: %s", err) + } + + return writeCertificateForKey(cmd, keyPath, fh, debugLogger) +} + +func writeCertificateForKey(cmd *cobra.Command, keyPath string, fh file.Handler, debugLogger debugLog) error { + // NOTE(miampf): Since other KMS aren't fully implemented yet, this commands assumes that the cKMS is used and derives the key accordingly. + var mastersecret uri.MasterSecret + if err := fh.ReadJSON(constants.MasterSecretFilename, &mastersecret); err != nil { + return fmt.Errorf("reading master secret (does %q exist?): %w", constants.MasterSecretFilename, err) + } + + mastersecretURI := uri.MasterSecret{Key: mastersecret.Key, Salt: mastersecret.Salt} + kms, err := setup.KMS(cmd.Context(), uri.NoStoreURI, mastersecretURI.EncodeToURI()) + if err != nil { + return fmt.Errorf("setting up KMS: %s", err) + } + sshCAKeySeed, err := kms.GetDEK(cmd.Context(), crypto.DEKPrefix+constants.SSHCAKeySuffix, ed25519.SeedSize) + if err != nil { + return fmt.Errorf("retrieving key from KMS: %s", err) + } + + ca, err := crypto.GenerateEmergencySSHCAKey(sshCAKeySeed) + if err != nil { + return fmt.Errorf("generating SSH emergency CA key: %s", err) + } + + marshalledKey := string(ssh.MarshalAuthorizedKey(ca.PublicKey())) + debugLogger.Debug("SSH CA KEY generated", "public-key", marshalledKey) + knownHostsContent := fmt.Sprintf("@cert-authority * %s", marshalledKey) + if err := fh.Write("./known_hosts", []byte(knownHostsContent), file.OptMkdirAll); err != nil { + return fmt.Errorf("writing known hosts file: %w", err) + } + + keyBuffer, err := fh.Read(keyPath) + if err != nil { + return fmt.Errorf("reading public key %q: %s", keyPath, err) + } + + pub, _, _, _, err := ssh.ParseAuthorizedKey(keyBuffer) + if err != nil { + return fmt.Errorf("parsing public key %q: %s", keyPath, err) + } + + certificate := ssh.Certificate{ + Key: pub, + CertType: ssh.UserCert, + ValidAfter: uint64(time.Now().Unix()), + ValidBefore: uint64(time.Now().Add(24 * time.Hour).Unix()), + ValidPrincipals: []string{"root"}, + Permissions: ssh.Permissions{ + Extensions: map[string]string{ + "permit-port-forwarding": "", + "permit-pty": "", + }, + }, + } + if err := certificate.SignCert(rand.Reader, ca); err != nil { + return fmt.Errorf("signing certificate: %s", err) + } + + debugLogger.Debug("Signed certificate", "certificate", string(ssh.MarshalAuthorizedKey(&certificate))) + if err := fh.Write("constellation_cert.pub", ssh.MarshalAuthorizedKey(&certificate), file.OptOverwrite); err != nil { + return fmt.Errorf("writing certificate: %s", err) + } + cmd.Printf("You can now connect to a node using the \"constellation_cert.pub\" certificate.\nLook at the documentation for a how-to guide:\n\n\thttps://docs.edgeless.systems/constellation/workflows/troubleshooting#emergency-ssh-access\n") + + return nil +} diff --git a/cli/internal/cmd/ssh_test.go b/cli/internal/cmd/ssh_test.go new file mode 100644 index 000000000..c5ba77c2c --- /dev/null +++ b/cli/internal/cmd/ssh_test.go @@ -0,0 +1,95 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package cmd + +import ( + "bytes" + "testing" + + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/logger" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" +) + +func TestSSH(t *testing.T) { + someSSHPubKey := "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDA1yYg1PIJNjAGjyuv66r8AJtpfBDFLdp3u9lVwkgbVKv1AzcaeTF/NEw+nhNJOjuCZ61LTPj12LZ8Wy/oSm0A= motte@lolcatghost" + someSSHPubKeyPath := "some-key.pub" + someMasterSecret := ` + { + "key": "MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAK", + "salt": "MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAK" + } + ` + testCases := map[string]struct { + fh file.Handler + pubKey string + masterSecret string + wantErr bool + }{ + "everything exists": { + fh: file.NewHandler(afero.NewMemMapFs()), + pubKey: someSSHPubKey, + masterSecret: someMasterSecret, + }, + "no public key": { + fh: file.NewHandler(afero.NewMemMapFs()), + masterSecret: someMasterSecret, + wantErr: true, + }, + "no master secret": { + fh: file.NewHandler(afero.NewMemMapFs()), + pubKey: someSSHPubKey, + wantErr: true, + }, + "malformed public key": { + fh: file.NewHandler(afero.NewMemMapFs()), + pubKey: "asdf", + masterSecret: someMasterSecret, + wantErr: true, + }, + "malformed master secret": { + fh: file.NewHandler(afero.NewMemMapFs()), + masterSecret: "asdf", + pubKey: someSSHPubKey, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + if tc.pubKey != "" { + require.NoError(tc.fh.Write(someSSHPubKeyPath, []byte(tc.pubKey))) + } + if tc.masterSecret != "" { + require.NoError(tc.fh.Write(constants.MasterSecretFilename, []byte(tc.masterSecret))) + } + + cmd := NewSSHCmd() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetIn(&bytes.Buffer{}) + + err := writeCertificateForKey(cmd, someSSHPubKeyPath, tc.fh, logger.NewTest(t)) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + cert, err := tc.fh.Read("constellation_cert.pub") + require.NoError(err) + _, _, _, _, err = ssh.ParseAuthorizedKey(cert) + require.NoError(err) + } + }) + } +} diff --git a/cli/internal/cmd/status.go b/cli/internal/cmd/status.go index 73f24ffba..5d9051c63 100644 --- a/cli/internal/cmd/status.go +++ b/cli/internal/cmd/status.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -43,7 +43,6 @@ func runStatus(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() fileHandler := file.NewHandler(afero.NewOsFs()) diff --git a/cli/internal/cmd/status_test.go b/cli/internal/cmd/status_test.go index 7984afbe8..e46bae917 100644 --- a/cli/internal/cmd/status_test.go +++ b/cli/internal/cmd/status_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -19,7 +19,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constellation/kubecmd" "github.com/edgelesssys/constellation/v2/internal/file" - updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/cli/internal/cmd/terminate.go b/cli/internal/cmd/terminate.go index c20c3fe2c..a194358ea 100644 --- a/cli/internal/cmd/terminate.go +++ b/cli/internal/cmd/terminate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/terminate_test.go b/cli/internal/cmd/terminate_test.go index e2888d4b0..24c9ee717 100644 --- a/cli/internal/cmd/terminate_test.go +++ b/cli/internal/cmd/terminate_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -104,7 +104,7 @@ func TestTerminate(t *testing.T) { }, "missing id file does not error": { stateFile: state.New(), - setupFs: func(require *require.Assertions, stateFile *state.State) afero.Fs { + setupFs: func(require *require.Assertions, _ *state.State) afero.Fs { fs := afero.NewMemMapFs() fileHandler := file.NewHandler(fs) require.NoError(fileHandler.Write(constants.AdminConfFilename, []byte{1, 2}, file.OptNone)) diff --git a/cli/internal/cmd/upgrade.go b/cli/internal/cmd/upgrade.go index 21addcb06..6c7db4966 100644 --- a/cli/internal/cmd/upgrade.go +++ b/cli/internal/cmd/upgrade.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/upgradeapply.go b/cli/internal/cmd/upgradeapply.go index a87e4b2c5..8f2465cbd 100644 --- a/cli/internal/cmd/upgradeapply.go +++ b/cli/internal/cmd/upgradeapply.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/upgradeapply_test.go b/cli/internal/cmd/upgradeapply_test.go index 68f740753..0062d444d 100644 --- a/cli/internal/cmd/upgradeapply_test.go +++ b/cli/internal/cmd/upgradeapply_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -254,7 +254,7 @@ func TestUpgradeApply(t *testing.T) { log: logger.NewTest(t), spinner: &nopSpinner{}, merger: &stubMerger{}, - newInfraApplier: func(ctx context.Context) (cloudApplier, func(), error) { + newInfraApplier: func(_ context.Context) (cloudApplier, func(), error) { return tc.terraformUpgrader, func() {}, nil }, applier: &stubConstellApplier{ @@ -335,7 +335,7 @@ func (u stubTerraformUpgrader) Plan(_ context.Context, _ *config.Config) (bool, return u.terraformDiff, u.planTerraformErr } -func (u stubTerraformUpgrader) Apply(_ context.Context, _ cloudprovider.Provider, _ cloudcmd.RollbackBehavior) (state.Infrastructure, error) { +func (u stubTerraformUpgrader) Apply(_ context.Context, _ cloudprovider.Provider, _ variant.Variant, _ cloudcmd.RollbackBehavior) (state.Infrastructure, error) { return state.Infrastructure{}, u.applyTerraformErr } @@ -356,8 +356,8 @@ func (m *mockTerraformUpgrader) Plan(ctx context.Context, conf *config.Config) ( return args.Bool(0), args.Error(1) } -func (m *mockTerraformUpgrader) Apply(ctx context.Context, provider cloudprovider.Provider, rollback cloudcmd.RollbackBehavior) (state.Infrastructure, error) { - args := m.Called(ctx, provider, rollback) +func (m *mockTerraformUpgrader) Apply(ctx context.Context, provider cloudprovider.Provider, variant variant.Variant, rollback cloudcmd.RollbackBehavior) (state.Infrastructure, error) { + args := m.Called(ctx, provider, variant, rollback) return args.Get(0).(state.Infrastructure), args.Error(1) } @@ -375,10 +375,18 @@ type mockApplier struct { mock.Mock } +func (m *mockApplier) AnnotateCoreDNSResources(_ context.Context) error { + return nil +} + +func (m *mockApplier) CleanupCoreDNSResources(_ context.Context) error { + return nil +} + func (m *mockApplier) PrepareHelmCharts( - helmOpts helm.Options, stateFile *state.State, str string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig, + helmOpts helm.Options, stateFile *state.State, str string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) { - args := m.Called(helmOpts, stateFile, helmOpts, str, masterSecret, openStackCfg) + args := m.Called(helmOpts, stateFile, helmOpts, str, masterSecret) return args.Get(0).(helm.Applier), args.Bool(1), args.Error(2) } diff --git a/cli/internal/cmd/upgradecheck.go b/cli/internal/cmd/upgradecheck.go index 491648074..570f5375f 100644 --- a/cli/internal/cmd/upgradecheck.go +++ b/cli/internal/cmd/upgradecheck.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -92,7 +92,6 @@ func runUpgradeCheck(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() var flags upgradeCheckFlags if err := flags.parse(cmd.Flags()); err != nil { @@ -188,7 +187,7 @@ func (u *upgradeCheckCmd) upgradeCheck(cmd *cobra.Command, fetcher attestationco // get current image version of the cluster csp := conf.GetProvider() attestationVariant := conf.GetAttestationConfig().GetVariant() - u.log.Debugf("Using provider %s with attestation variant %s", csp.String(), attestationVariant.String()) + u.log.Debug(fmt.Sprintf("Using provider %q with attestation variant %q", csp.String(), attestationVariant.String())) current, err := u.collect.currentVersions(cmd.Context()) if err != nil { @@ -199,18 +198,18 @@ func (u *upgradeCheckCmd) upgradeCheck(cmd *cobra.Command, fetcher attestationco if err != nil { return err } - u.log.Debugf("Current cli version: %s", current.cli) - u.log.Debugf("Supported cli version(s): %s", supported.cli) - u.log.Debugf("Current service version: %s", current.service) - u.log.Debugf("Supported service version: %s", supported.service) - u.log.Debugf("Current k8s version: %s", current.k8s) - u.log.Debugf("Supported k8s version(s): %s", supported.k8s) + u.log.Debug(fmt.Sprintf("Current cli version: %q", current.cli)) + u.log.Debug(fmt.Sprintf("Supported cli version(s): %q", supported.cli)) + u.log.Debug(fmt.Sprintf("Current service version: %q", current.service)) + u.log.Debug(fmt.Sprintf("Supported service version: %q", supported.service)) + u.log.Debug(fmt.Sprintf("Current k8s version: %q", current.k8s)) + u.log.Debug(fmt.Sprintf("Supported k8s version(s): %q", supported.k8s)) // Filter versions to only include upgrades newServices := supported.service if err := supported.service.IsUpgradeTo(current.service); err != nil { newServices = consemver.Semver{} - u.log.Debugf("No valid service upgrades are available from %q to %q. The minor version can only drift by 1.\n", current.service.String(), supported.service.String()) + u.log.Debug(fmt.Sprintf("No valid service upgrades are available from %q to %q. The minor version can only drift by 1.\n", current.service.String(), supported.service.String())) } newKubernetes := filterK8sUpgrades(current.k8s, supported.k8s) @@ -222,13 +221,13 @@ func (u *upgradeCheckCmd) upgradeCheck(cmd *cobra.Command, fetcher attestationco return err } - u.log.Debugf("Planning Terraform migrations") + u.log.Debug("Planning Terraform migrations") // Add manual migrations here if required // // var manualMigrations []terraform.StateMigration // for _, migration := range manualMigrations { - // u.log.Debugf("Adding manual Terraform migration: %s", migration.DisplayName) + // u.log.Debug("Adding manual Terraform migration: %s", migration.DisplayName) // u.terraformChecker.AddManualStateMigration(migration) // } cmd.Println("The following Terraform migrations are available with this CLI:") @@ -344,7 +343,7 @@ func (v *versionCollector) newMeasurements(ctx context.Context, csp cloudprovide // get expected measurements for each image upgrades := make(map[string]measurements.M) for _, version := range versions { - v.log.Debugf("Fetching measurements for image: %s", version) + v.log.Debug(fmt.Sprintf("Fetching measurements for image: %q", version.Version())) shortPath := version.ShortPath() publicKey, err := keyselect.CosignPublicKeyForVersion(version) @@ -364,8 +363,8 @@ func (v *versionCollector) newMeasurements(ctx context.Context, csp cloudprovide continue } upgrades[shortPath] = measurements + v.log.Debug("Compatible image measurement found", shortPath, measurements.String()) } - v.log.Debugf("Compatible image measurements are %v", upgrades) return upgrades, nil } @@ -453,9 +452,9 @@ func (v *versionCollector) newImages(ctx context.Context, currentImageVersion co if err != nil { return nil, fmt.Errorf("calculating next image minor version: %w", err) } - v.log.Debugf("Current image minor version is %s", currentImageMinorVer) - v.log.Debugf("Current CLI minor version is %s", currentCLIMinorVer) - v.log.Debugf("Next image minor version is %s", nextImageMinorVer) + v.log.Debug(fmt.Sprintf("Current image minor version is %q", currentImageMinorVer)) + v.log.Debug(fmt.Sprintf("Current CLI minor version is %q", currentCLIMinorVer)) + v.log.Debug(fmt.Sprintf("Next image minor version is %q", nextImageMinorVer)) allowedMinorVersions := []string{currentImageMinorVer, nextImageMinorVer} switch cliImageCompare := semver.Compare(currentCLIMinorVer, currentImageMinorVer); { @@ -471,7 +470,7 @@ func (v *versionCollector) newImages(ctx context.Context, currentImageVersion co case cliImageCompare > 0: allowedMinorVersions = []string{currentImageMinorVer, nextImageMinorVer} } - v.log.Debugf("Allowed minor versions are %#v", allowedMinorVersions) + v.log.Debug(fmt.Sprintf("Allowed minor versions are %#v", allowedMinorVersions)) newerImages, err := v.newerVersions(ctx, allowedMinorVersions) if err != nil { @@ -494,7 +493,7 @@ func (v *versionCollector) newerVersions(ctx context.Context, allowedVersions [] patchList, err := v.verListFetcher.FetchVersionList(ctx, patchList) var notFound *fetcher.NotFoundError if errors.As(err, ¬Found) { - v.log.Debugf("Skipping version: %s", err) + v.log.Debug(fmt.Sprintf("Skipping version: %q", err)) continue } if err != nil { @@ -502,7 +501,7 @@ func (v *versionCollector) newerVersions(ctx context.Context, allowedVersions [] } updateCandidates = append(updateCandidates, patchList.StructuredVersions()...) } - v.log.Debugf("Update candidates are %v", updateCandidates) + v.log.Debug(fmt.Sprintf("Update candidates are %v", updateCandidates)) return updateCandidates, nil } @@ -604,7 +603,7 @@ func getCompatibleImageMeasurements(ctx context.Context, writer io.Writer, clien } var fetchedMeasurements measurements.M - log.Debugf("Fetching for measurement url: %s", measurementsURL) + log.Debug(fmt.Sprintf("Fetching for measurement url: %q", measurementsURL)) hash, err := fetchedMeasurements.FetchAndVerify( ctx, client, cosign, @@ -658,7 +657,7 @@ func (v *versionCollector) newCLIVersions(ctx context.Context) ([]consemver.Semv return nil, fmt.Errorf("parsing version %s: %w", version, err) } if err := target.IsUpgradeTo(v.cliVersion); err != nil { - v.log.Debugf("Skipping incompatible minor version %q: %s", version, err) + v.log.Debug(fmt.Sprintf("Skipping incompatible minor version %q: %q", version, err)) continue } list := versionsapi.List{ @@ -692,7 +691,7 @@ func (v *versionCollector) filterCompatibleCLIVersions(ctx context.Context, cliP var compatibleVersions []consemver.Semver for _, version := range cliPatchVersions { if err := version.IsUpgradeTo(v.cliVersion); err != nil { - v.log.Debugf("Skipping incompatible patch version %q: %s", version, err) + v.log.Debug(fmt.Sprintf("Skipping incompatible patch version %q: %q", version, err)) continue } req := versionsapi.CLIInfo{ diff --git a/cli/internal/cmd/upgradecheck_test.go b/cli/internal/cmd/upgradecheck_test.go index 5e6f8329a..19020fc0d 100644 --- a/cli/internal/cmd/upgradecheck_test.go +++ b/cli/internal/cmd/upgradecheck_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -139,7 +139,7 @@ func TestGetCompatibleImageMeasurements(t *testing.T) { } }) - upgrades, err := getCompatibleImageMeasurements(context.Background(), &bytes.Buffer{}, client, &stubCosignVerifier{}, singleUUIDVerifier(), csp, attestationVariant, versionZero, logger.NewTest(t)) + upgrades, err := getCompatibleImageMeasurements(t.Context(), &bytes.Buffer{}, client, &stubCosignVerifier{}, singleUUIDVerifier(), csp, attestationVariant, versionZero, logger.NewTest(t)) assert.NoError(err) for _, measurement := range upgrades { @@ -344,7 +344,7 @@ func TestNewCLIVersions(t *testing.T) { t.Run(name, func(t *testing.T) { require := require.New(t) - _, err := tc.verCollector.newCLIVersions(context.Background()) + _, err := tc.verCollector.newCLIVersions(t.Context()) if tc.wantErr { require.Error(err) return @@ -385,7 +385,7 @@ func TestFilterCompatibleCLIVersions(t *testing.T) { t.Run(name, func(t *testing.T) { require := require.New(t) - _, err := tc.verCollector.filterCompatibleCLIVersions(context.Background(), tc.cliPatchVersions, consemver.NewFromInt(1, 24, 5, "")) + _, err := tc.verCollector.filterCompatibleCLIVersions(t.Context(), tc.cliPatchVersions, consemver.NewFromInt(1, 24, 5, "")) if tc.wantErr { require.Error(err) return diff --git a/cli/internal/cmd/userinteraction.go b/cli/internal/cmd/userinteraction.go index 4fef6256e..cef1921a4 100644 --- a/cli/internal/cmd/userinteraction.go +++ b/cli/internal/cmd/userinteraction.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/userinteraction_test.go b/cli/internal/cmd/userinteraction_test.go index 77e541b50..91472c08b 100644 --- a/cli/internal/cmd/userinteraction_test.go +++ b/cli/internal/cmd/userinteraction_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -23,7 +23,7 @@ func TestAskToConfirm(t *testing.T) { cmd := &cobra.Command{ Use: "test", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ok, err := askToConfirm(cmd, "777") if err != nil { return err diff --git a/cli/internal/cmd/validargs.go b/cli/internal/cmd/validargs.go index 73b84c6e4..1c83ae3bf 100644 --- a/cli/internal/cmd/validargs.go +++ b/cli/internal/cmd/validargs.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -14,7 +14,7 @@ import ( ) func isCloudProvider(arg int) cobra.PositionalArgs { - return func(cmd *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { if provider := cloudprovider.FromString(args[arg]); provider == cloudprovider.Unknown { return fmt.Errorf("argument %s isn't a valid cloud provider", args[arg]) } diff --git a/cli/internal/cmd/validargs_test.go b/cli/internal/cmd/validargs_test.go index 63d783e9e..f29d179e6 100644 --- a/cli/internal/cmd/validargs_test.go +++ b/cli/internal/cmd/validargs_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/verifier_test.go b/cli/internal/cmd/verifier_test.go index b55c0ab15..fcb27ab2b 100644 --- a/cli/internal/cmd/verifier_test.go +++ b/cli/internal/cmd/verifier_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/verify.go b/cli/internal/cmd/verify.go index b9f7708a4..2e9afba11 100644 --- a/cli/internal/cmd/verify.go +++ b/cli/internal/cmd/verify.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -21,16 +21,14 @@ import ( "strconv" "strings" - tpmProto "github.com/google/go-tpm-tools/proto/tpm" - "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" "github.com/edgelesssys/constellation/v2/internal/atls" + azuretdx "github.com/edgelesssys/constellation/v2/internal/attestation/azure/tdx" "github.com/edgelesssys/constellation/v2/internal/attestation/choose" "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/snp" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constellation/state" @@ -39,6 +37,12 @@ import ( "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/verify" "github.com/edgelesssys/constellation/v2/verify/verifyproto" + + "github.com/google/go-sev-guest/proto/sevsnp" + "github.com/google/go-tdx-guest/abi" + "github.com/google/go-tdx-guest/proto/tdx" + "github.com/google/go-tpm-tools/proto/attest" + tpmProto "github.com/google/go-tpm-tools/proto/tpm" "github.com/spf13/afero" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -101,28 +105,13 @@ func runVerify(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("creating logger: %w", err) } - defer log.Sync() fileHandler := file.NewHandler(afero.NewOsFs()) verifyClient := &constellationVerifier{ - dialer: dialer.New(nil, nil, &net.Dialer{}), + dialer: dialer.New(nil, nil, nil), log: log, } - formatterFactory := func(output string, provider cloudprovider.Provider, log debugLog) (attestationDocFormatter, error) { - if output == "json" && (provider != cloudprovider.Azure && provider != cloudprovider.AWS) { - return nil, errors.New("json output is only supported for Azure and AWS") - } - switch output { - case "json": - return &jsonAttestationDocFormatter{log}, nil - case "raw": - return &rawAttestationDocFormatter{log}, nil - case "": - return &defaultAttestationDocFormatter{log}, nil - default: - return nil, fmt.Errorf("invalid output value for formatter: %s", output) - } - } + v := &verifyCmd{ fileHandler: fileHandler, log: log, @@ -130,15 +119,14 @@ func runVerify(cmd *cobra.Command, _ []string) error { if err := v.flags.parse(cmd.Flags()); err != nil { return err } - v.log.Debugf("Using flags: %+v", v.flags) + v.log.Debug("Using flags", "clusterID", v.flags.clusterID, "endpoint", v.flags.endpoint, "ownerID", v.flags.ownerID) + fetcher := attestationconfigapi.NewFetcher() - return v.verify(cmd, verifyClient, formatterFactory, fetcher) + return v.verify(cmd, verifyClient, fetcher) } -type formatterFactory func(output string, provider cloudprovider.Provider, log debugLog) (attestationDocFormatter, error) - -func (c *verifyCmd) verify(cmd *cobra.Command, verifyClient verifyClient, factory formatterFactory, configFetcher attestationconfigapi.Fetcher) error { - c.log.Debugf("Loading configuration file from %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename)) +func (c *verifyCmd) verify(cmd *cobra.Command, verifyClient verifyClient, configFetcher attestationconfigapi.Fetcher) error { + c.log.Debug(fmt.Sprintf("Loading configuration file from %q", c.flags.pathPrefixer.PrefixPrintablePath(constants.ConfigFilename))) conf, err := config.New(c.fileHandler, constants.ConfigFilename, configFetcher, c.flags.force) var configValidationErr *config.ValidationError if errors.As(err, &configValidationErr) { @@ -150,10 +138,7 @@ func (c *verifyCmd) verify(cmd *cobra.Command, verifyClient verifyClient, factor stateFile, err := state.ReadFromFile(c.fileHandler, constants.StateFilename) if err != nil { - return fmt.Errorf("reading state file: %w", err) - } - if err := stateFile.Validate(state.PostInit, conf.GetProvider()); err != nil { - return fmt.Errorf("validating state file: %w", err) + stateFile = state.New() // A state file is only required if the user has not provided IP or ID flags } ownerID, clusterID, err := c.validateIDFlags(cmd, stateFile) @@ -171,13 +156,13 @@ func (c *verifyCmd) verify(cmd *cobra.Command, verifyClient verifyClient, factor } conf.UpdateMAAURL(maaURL) - c.log.Debugf("Updating expected PCRs") + c.log.Debug("Updating expected PCRs") attConfig := conf.GetAttestationConfig() if err := updateInitMeasurements(attConfig, ownerID, clusterID); err != nil { return fmt.Errorf("updating expected PCRs: %w", err) } - c.log.Debugf("Creating aTLS Validator for %s", conf.GetAttestationConfig().GetVariant()) + c.log.Debug(fmt.Sprintf("Creating aTLS Validator for %q", conf.GetAttestationConfig().GetVariant())) validator, err := choose.Validator(attConfig, warnLogger{cmd: cmd, log: c.log}) if err != nil { return fmt.Errorf("creating aTLS validator: %w", err) @@ -187,7 +172,7 @@ func (c *verifyCmd) verify(cmd *cobra.Command, verifyClient verifyClient, factor if err != nil { return fmt.Errorf("generating random nonce: %w", err) } - c.log.Debugf("Generated random nonce: %x", nonce) + c.log.Debug(fmt.Sprintf("Generated random nonce: %x", nonce)) rawAttestationDoc, err := verifyClient.Verify( cmd.Context(), @@ -201,20 +186,27 @@ func (c *verifyCmd) verify(cmd *cobra.Command, verifyClient verifyClient, factor return fmt.Errorf("verifying: %w", err) } - // certificates are only available for Azure - formatter, err := factory(c.flags.output, conf.GetProvider(), c.log) - if err != nil { - return fmt.Errorf("creating formatter: %w", err) - } - attDocOutput, err := formatter.format( - cmd.Context(), - rawAttestationDoc, - (conf.Provider.Azure == nil && conf.Provider.AWS == nil), - attConfig, - ) - if err != nil { - return fmt.Errorf("printing attestation document: %w", err) + var attDocOutput string + switch c.flags.output { + case "json": + attDocOutput, err = formatJSON(cmd.Context(), rawAttestationDoc, attConfig, c.log) + if err != nil { + return fmt.Errorf("printing attestation document: %w", err) + } + + case "raw": + attDocOutput = fmt.Sprintf("Attestation Document:\n%s\n", rawAttestationDoc) + + case "": + attDocOutput, err = formatDefault(cmd.Context(), rawAttestationDoc, attConfig, c.log) + if err != nil { + return fmt.Errorf("printing attestation document: %w", err) + } + + default: + return fmt.Errorf("invalid output value for formatter: %s", c.flags.output) } + cmd.Println(attDocOutput) cmd.PrintErrln("Verification OK") @@ -254,82 +246,92 @@ func (c *verifyCmd) validateEndpointFlag(cmd *cobra.Command, stateFile *state.St return endpoint, nil } -// an attestationDocFormatter formats the attestation document. -type attestationDocFormatter interface { - // format returns the raw or formatted attestation doc depending on the rawOutput argument. - format(ctx context.Context, docString string, PCRsOnly bool, attestationCfg config.AttestationCfg) (string, error) -} - -type jsonAttestationDocFormatter struct { - log debugLog -} - -// format returns the json formatted attestation doc. -func (f *jsonAttestationDocFormatter) format(ctx context.Context, docString string, _ bool, - attestationCfg config.AttestationCfg, +// formatJSON returns the json formatted attestation doc. +func formatJSON(ctx context.Context, docString []byte, attestationCfg config.AttestationCfg, log debugLog, ) (string, error) { - var doc attestationDoc - if err := json.Unmarshal([]byte(docString), &doc); err != nil { - return "", fmt.Errorf("unmarshal attestation document: %w", err) + doc, err := unmarshalAttDoc(docString, attestationCfg.GetVariant()) + if err != nil { + return "", fmt.Errorf("unmarshalling attestation document: %w", err) } - instanceInfo, err := extractInstanceInfo(doc) - if err != nil { + switch attestationCfg.GetVariant() { + case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}: + return snpFormatJSON(ctx, doc.InstanceInfo, attestationCfg, log) + case variant.AzureTDX{}: + return tdxFormatJSON(doc.InstanceInfo, attestationCfg) + default: + return "", fmt.Errorf("json output is not supported for variant %s", attestationCfg.GetVariant()) + } +} + +func snpFormatJSON(ctx context.Context, instanceInfoRaw []byte, attestationCfg config.AttestationCfg, log debugLog, +) (string, error) { + var instanceInfo snp.InstanceInfo + if err := json.Unmarshal(instanceInfoRaw, &instanceInfo); err != nil { return "", fmt.Errorf("unmarshalling instance info: %w", err) } - report, err := verify.NewReport(ctx, instanceInfo, attestationCfg, f.log) + report, err := verify.NewReport(ctx, instanceInfo, attestationCfg, log) if err != nil { return "", fmt.Errorf("parsing SNP report: %w", err) } jsonBytes, err := json.Marshal(report) - return string(jsonBytes), err } -type rawAttestationDocFormatter struct { - log debugLog -} +func tdxFormatJSON(instanceInfoRaw []byte, attestationCfg config.AttestationCfg) (string, error) { + var rawQuote []byte -// format returns the raw attestation doc. -func (f *rawAttestationDocFormatter) format(_ context.Context, docString string, _ bool, - _ config.AttestationCfg, -) (string, error) { - b := &strings.Builder{} - b.WriteString("Attestation Document:\n") - b.WriteString(fmt.Sprintf("%s\n", docString)) - return b.String(), nil -} + if attestationCfg.GetVariant().Equal(variant.AzureTDX{}) { + var instanceInfo azuretdx.InstanceInfo + if err := json.Unmarshal(instanceInfoRaw, &instanceInfo); err != nil { + return "", fmt.Errorf("unmarshalling instance info: %w", err) + } + rawQuote = instanceInfo.AttestationReport + } -type defaultAttestationDocFormatter struct { - log debugLog + tdxQuote, err := abi.QuoteToProto(rawQuote) + if err != nil { + return "", fmt.Errorf("converting quote to proto: %w", err) + } + quote, ok := tdxQuote.(*tdx.QuoteV4) + if !ok { + return "", fmt.Errorf("unexpected quote type: %T", tdxQuote) + } + + quoteJSON, err := json.Marshal(quote) + return string(quoteJSON), err } // format returns the formatted attestation doc. -func (f *defaultAttestationDocFormatter) format(ctx context.Context, docString string, PCRsOnly bool, - attestationCfg config.AttestationCfg, +func formatDefault(ctx context.Context, docString []byte, attestationCfg config.AttestationCfg, log debugLog, ) (string, error) { b := &strings.Builder{} b.WriteString("Attestation Document:\n") - var doc attestationDoc - if err := json.Unmarshal([]byte(docString), &doc); err != nil { - return "", fmt.Errorf("unmarshal attestation document: %w", err) + doc, err := unmarshalAttDoc(docString, attestationCfg.GetVariant()) + if err != nil { + return "", fmt.Errorf("unmarshalling attestation document: %w", err) } - if err := f.parseQuotes(b, doc.Attestation.Quotes, attestationCfg.GetMeasurements()); err != nil { + if err := parseQuotes(b, doc.Attestation.Quotes, attestationCfg.GetMeasurements()); err != nil { return "", fmt.Errorf("parse quote: %w", err) } - if PCRsOnly { + + // If we have a non SNP variant, print only the PCRs + if !(attestationCfg.GetVariant().Equal(variant.AzureSEVSNP{}) || + attestationCfg.GetVariant().Equal(variant.AWSSEVSNP{}) || + attestationCfg.GetVariant().Equal(variant.GCPSEVSNP{})) { return b.String(), nil } - instanceInfo, err := extractInstanceInfo(doc) - if err != nil { + // SNP reports contain extra information that we can print + var instanceInfo snp.InstanceInfo + if err := json.Unmarshal(doc.InstanceInfo, &instanceInfo); err != nil { return "", fmt.Errorf("unmarshalling instance info: %w", err) } - report, err := verify.NewReport(ctx, instanceInfo, attestationCfg, f.log) + report, err := verify.NewReport(ctx, instanceInfo, attestationCfg, log) if err != nil { return "", fmt.Errorf("parsing SNP report: %w", err) } @@ -338,7 +340,7 @@ func (f *defaultAttestationDocFormatter) format(ctx context.Context, docString s } // parseQuotes parses the base64-encoded quotes and writes their details to the output builder. -func (f *defaultAttestationDocFormatter) parseQuotes(b *strings.Builder, quotes []*tpmProto.Quote, expectedPCRs measurements.M) error { +func parseQuotes(b *strings.Builder, quotes []*tpmProto.Quote, expectedPCRs measurements.M) error { writeIndentfln(b, 1, "Quote:") var pcrNumbers []uint32 @@ -365,18 +367,6 @@ func (f *defaultAttestationDocFormatter) parseQuotes(b *strings.Builder, quotes return nil } -// attestationDoc is the attestation document returned by the verifier. -type attestationDoc struct { - Attestation struct { - AkPub string `json:"ak_pub"` - Quotes []*tpmProto.Quote `json:"quotes"` - EventLog string `json:"event_log"` - TeeAttestation interface{} `json:"TeeAttestation"` - } `json:"Attestation"` - InstanceInfo string `json:"InstanceInfo"` - UserData string `json:"UserData"` -} - type constellationVerifier struct { dialer grpcInsecureDialer log debugLog @@ -385,41 +375,41 @@ type constellationVerifier struct { // Verify retrieves an attestation statement from the Constellation and verifies it using the validator. func (v *constellationVerifier) Verify( ctx context.Context, endpoint string, req *verifyproto.GetAttestationRequest, validator atls.Validator, -) (string, error) { - v.log.Debugf("Dialing endpoint: %q", endpoint) - conn, err := v.dialer.DialInsecure(ctx, endpoint) +) ([]byte, error) { + v.log.Debug(fmt.Sprintf("Dialing endpoint: %q", endpoint)) + conn, err := v.dialer.DialInsecure(endpoint) if err != nil { - return "", fmt.Errorf("dialing init server: %w", err) + return nil, fmt.Errorf("dialing init server: %w", err) } defer conn.Close() client := verifyproto.NewAPIClient(conn) - v.log.Debugf("Sending attestation request") + v.log.Debug("Sending attestation request") resp, err := client.GetAttestation(ctx, req) if err != nil { - return "", fmt.Errorf("getting attestation: %w", err) + return nil, fmt.Errorf("getting attestation: %w", err) } - v.log.Debugf("Verifying attestation") + v.log.Debug("Verifying attestation") signedData, err := validator.Validate(ctx, resp.Attestation, req.Nonce) if err != nil { - return "", fmt.Errorf("validating attestation: %w", err) + return nil, fmt.Errorf("validating attestation: %w", err) } if !bytes.Equal(signedData, []byte(constants.ConstellationVerifyServiceUserData)) { - return "", errors.New("signed data in attestation does not match expected user data") + return nil, errors.New("signed data in attestation does not match expected user data") } - return string(resp.Attestation), nil + return resp.Attestation, nil } type verifyClient interface { - Verify(ctx context.Context, endpoint string, req *verifyproto.GetAttestationRequest, validator atls.Validator) (string, error) + Verify(ctx context.Context, endpoint string, req *verifyproto.GetAttestationRequest, validator atls.Validator) ([]byte, error) } type grpcInsecureDialer interface { - DialInsecure(ctx context.Context, endpoint string) (conn *grpc.ClientConn, err error) + DialInsecure(endpoint string) (conn *grpc.ClientConn, err error) } // writeIndentfln writes a formatted string to the builder with the given indentation level @@ -431,19 +421,6 @@ func writeIndentfln(b *strings.Builder, indentLvl int, format string, args ...an b.WriteString(fmt.Sprintf(format+"\n", args...)) } -func extractInstanceInfo(doc attestationDoc) (snp.InstanceInfo, error) { - instanceInfoString, err := base64.StdEncoding.DecodeString(doc.InstanceInfo) - if err != nil { - return snp.InstanceInfo{}, fmt.Errorf("decode instance info: %w", err) - } - - var instanceInfo snp.InstanceInfo - if err := json.Unmarshal(instanceInfoString, &instanceInfo); err != nil { - return snp.InstanceInfo{}, fmt.Errorf("unmarshal instance info: %w", err) - } - return instanceInfo, nil -} - func addPortIfMissing(endpoint string, defaultPort int) (string, error) { if endpoint == "" { return "", errors.New("endpoint is empty") @@ -467,7 +444,10 @@ func updateInitMeasurements(config config.AttestationCfg, ownerID, clusterID str m := config.GetMeasurements() switch config.GetVariant() { - case variant.AWSNitroTPM{}, variant.AWSSEVSNP{}, variant.AzureTrustedLaunch{}, variant.AzureSEVSNP{}, variant.GCPSEVES{}, variant.QEMUVTPM{}: + case variant.AWSNitroTPM{}, variant.AWSSEVSNP{}, + variant.AzureTrustedLaunch{}, variant.AzureSEVSNP{}, variant.AzureTDX{}, // AzureTDX also uses a vTPM for measurements + variant.GCPSEVES{}, variant.GCPSEVSNP{}, + variant.QEMUVTPM{}: if err := updateMeasurementTPM(m, uint32(measurements.PCRIndexOwnerID), ownerID); err != nil { return err } @@ -540,3 +520,26 @@ func decodeMeasurement(encoded string) ([]byte, error) { } return decoded, nil } + +func unmarshalAttDoc(attDocJSON []byte, attestationVariant variant.Variant) (vtpm.AttestationDocument, error) { + attDoc := vtpm.AttestationDocument{ + Attestation: &attest.Attestation{}, + } + + // Explicitly initialize this struct, as TeeAttestation + // is a "oneof" protobuf field, which needs an explicit + // type to be set to be unmarshaled correctly. + switch attestationVariant { + case variant.AzureTDX{}: + attDoc.Attestation.TeeAttestation = &attest.Attestation_TdxAttestation{ + TdxAttestation: &tdx.QuoteV4{}, + } + default: + attDoc.Attestation.TeeAttestation = &attest.Attestation_SevSnpAttestation{ + SevSnpAttestation: &sevsnp.Attestation{}, + } + } + + err := json.Unmarshal(attDocJSON, &attDoc) + return attDoc, err +} diff --git a/cli/internal/cmd/verify_test.go b/cli/internal/cmd/verify_test.go index 60a17aedc..4a140d8ed 100644 --- a/cli/internal/cmd/verify_test.go +++ b/cli/internal/cmd/verify_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -47,7 +47,6 @@ func TestVerify(t *testing.T) { testCases := map[string]struct { provider cloudprovider.Provider protoClient *stubVerifyClient - formatter *stubAttDocFormatter nodeEndpointFlag string clusterIDFlag string stateFile *state.State @@ -62,7 +61,6 @@ func TestVerify(t *testing.T) { protoClient: &stubVerifyClient{}, stateFile: defaultStateFile(cloudprovider.GCP), wantEndpoint: "192.0.2.1:1234", - formatter: &stubAttDocFormatter{}, }, "azure": { provider: cloudprovider.Azure, @@ -71,7 +69,6 @@ func TestVerify(t *testing.T) { protoClient: &stubVerifyClient{}, stateFile: defaultStateFile(cloudprovider.Azure), wantEndpoint: "192.0.2.1:1234", - formatter: &stubAttDocFormatter{}, }, "default port": { provider: cloudprovider.GCP, @@ -80,7 +77,6 @@ func TestVerify(t *testing.T) { protoClient: &stubVerifyClient{}, stateFile: defaultStateFile(cloudprovider.GCP), wantEndpoint: "192.0.2.1:" + strconv.Itoa(constants.VerifyServiceNodePortGRPC), - formatter: &stubAttDocFormatter{}, }, "endpoint not set": { provider: cloudprovider.GCP, @@ -91,8 +87,7 @@ func TestVerify(t *testing.T) { s.Infrastructure.ClusterEndpoint = "" return s }(), - formatter: &stubAttDocFormatter{}, - wantErr: true, + wantErr: true, }, "endpoint from state file": { provider: cloudprovider.GCP, @@ -104,7 +99,6 @@ func TestVerify(t *testing.T) { return s }(), wantEndpoint: "192.0.2.1:" + strconv.Itoa(constants.VerifyServiceNodePortGRPC), - formatter: &stubAttDocFormatter{}, }, "override endpoint from details file": { provider: cloudprovider.GCP, @@ -117,7 +111,6 @@ func TestVerify(t *testing.T) { return s }(), wantEndpoint: "192.0.2.2:1234", - formatter: &stubAttDocFormatter{}, }, "invalid endpoint": { provider: cloudprovider.GCP, @@ -125,7 +118,6 @@ func TestVerify(t *testing.T) { clusterIDFlag: zeroBase64, protoClient: &stubVerifyClient{}, stateFile: defaultStateFile(cloudprovider.GCP), - formatter: &stubAttDocFormatter{}, wantErr: true, }, "neither owner id nor cluster id set": { @@ -137,7 +129,6 @@ func TestVerify(t *testing.T) { s.ClusterValues.ClusterID = "" return s }(), - formatter: &stubAttDocFormatter{}, protoClient: &stubVerifyClient{}, wantErr: true, }, @@ -151,14 +142,12 @@ func TestVerify(t *testing.T) { return s }(), wantEndpoint: "192.0.2.1:1234", - formatter: &stubAttDocFormatter{}, }, "config file not existing": { provider: cloudprovider.GCP, clusterIDFlag: zeroBase64, nodeEndpointFlag: "192.0.2.1:1234", stateFile: defaultStateFile(cloudprovider.GCP), - formatter: &stubAttDocFormatter{}, skipConfigCreation: true, wantErr: true, }, @@ -168,7 +157,6 @@ func TestVerify(t *testing.T) { clusterIDFlag: zeroBase64, protoClient: &stubVerifyClient{verifyErr: rpcStatus.Error(codes.Internal, "failed")}, stateFile: defaultStateFile(cloudprovider.Azure), - formatter: &stubAttDocFormatter{}, wantErr: true, }, "error protoClient GetState not rpc": { @@ -177,18 +165,19 @@ func TestVerify(t *testing.T) { clusterIDFlag: zeroBase64, protoClient: &stubVerifyClient{verifyErr: someErr}, stateFile: defaultStateFile(cloudprovider.Azure), - formatter: &stubAttDocFormatter{}, wantErr: true, }, - "format error": { + "state file is not required if flags are given": { provider: cloudprovider.Azure, nodeEndpointFlag: "192.0.2.1:1234", clusterIDFlag: zeroBase64, protoClient: &stubVerifyClient{}, - stateFile: defaultStateFile(cloudprovider.Azure), wantEndpoint: "192.0.2.1:1234", - formatter: &stubAttDocFormatter{formatErr: someErr}, - wantErr: true, + }, + "no state file and no flags": { + provider: cloudprovider.Azure, + protoClient: &stubVerifyClient{}, + wantErr: true, }, } @@ -206,7 +195,9 @@ func TestVerify(t *testing.T) { cfg := defaultConfigWithExpectedMeasurements(t, config.Default(), tc.provider) require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, cfg)) } - require.NoError(tc.stateFile.WriteToFile(fileHandler, constants.StateFilename)) + if tc.stateFile != nil { + require.NoError(tc.stateFile.WriteToFile(fileHandler, constants.StateFilename)) + } v := &verifyCmd{ fileHandler: fileHandler, @@ -214,12 +205,10 @@ func TestVerify(t *testing.T) { flags: verifyFlags{ clusterID: tc.clusterIDFlag, endpoint: tc.nodeEndpointFlag, + output: "raw", }, } - formatterFac := func(_ string, _ cloudprovider.Provider, _ debugLog) (attestationDocFormatter, error) { - return tc.formatter, nil - } - err := v.verify(cmd, tc.protoClient, formatterFac, stubAttestationFetcher{}) + err := v.verify(cmd, tc.protoClient, stubAttestationFetcher{}) if tc.wantErr { assert.Error(err) } else { @@ -231,36 +220,22 @@ func TestVerify(t *testing.T) { } } -type stubAttDocFormatter struct { - formatErr error -} - -func (f *stubAttDocFormatter) format(_ context.Context, _ string, _ bool, _ config.AttestationCfg) (string, error) { - return "", f.formatErr -} - -func TestFormat(t *testing.T) { - formatter := func() *defaultAttestationDocFormatter { - return &defaultAttestationDocFormatter{ - log: logger.NewTest(t), - } - } - +func TestFormatDefault(t *testing.T) { testCases := map[string]struct { - formatter *defaultAttestationDocFormatter - doc string - wantErr bool + doc []byte + attCfg config.AttestationCfg + wantErr bool }{ "invalid doc": { - formatter: formatter(), - doc: "invalid", - wantErr: true, + doc: []byte("invalid"), + attCfg: &config.AzureSEVSNP{}, + wantErr: true, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - _, err := tc.formatter.format(context.Background(), tc.doc, false, nil) + _, err := formatDefault(t.Context(), tc.doc, tc.attCfg, logger.NewTest(t)) if tc.wantErr { assert.Error(t, err) } else { @@ -338,7 +313,7 @@ func TestVerifyClient(t *testing.T) { Nonce: tc.nonce, } - _, err = verifier.Verify(context.Background(), addr, request, atls.NewFakeValidator(variant.Dummy{})) + _, err = verifier.Verify(t.Context(), addr, request, atls.NewFakeValidator(variant.Dummy{})) if tc.wantErr { assert.Error(err) @@ -354,9 +329,9 @@ type stubVerifyClient struct { endpoint string } -func (c *stubVerifyClient) Verify(_ context.Context, endpoint string, _ *verifyproto.GetAttestationRequest, _ atls.Validator) (string, error) { +func (c *stubVerifyClient) Verify(_ context.Context, endpoint string, _ *verifyproto.GetAttestationRequest, _ atls.Validator) ([]byte, error) { c.endpoint = endpoint - return "", c.verifyErr + return nil, c.verifyErr } type stubVerifyAPI struct { @@ -502,9 +477,8 @@ func TestParseQuotes(t *testing.T) { assert := assert.New(t) b := &strings.Builder{} - parser := &defaultAttestationDocFormatter{} - err := parser.parseQuotes(b, tc.quotes, tc.expectedPCRs) + err := parseQuotes(b, tc.quotes, tc.expectedPCRs) if tc.wantErr { assert.Error(err) } else { diff --git a/cli/internal/cmd/version.go b/cli/internal/cmd/version.go index a61aee437..30ce98245 100644 --- a/cli/internal/cmd/version.go +++ b/cli/internal/cmd/version.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/cmd/version_test.go b/cli/internal/cmd/version_test.go index 646244423..f68041c66 100644 --- a/cli/internal/cmd/version_test.go +++ b/cli/internal/cmd/version_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/cli/internal/libvirt/BUILD.bazel b/cli/internal/libvirt/BUILD.bazel index d89676216..aa030ee5e 100644 --- a/cli/internal/libvirt/BUILD.bazel +++ b/cli/internal/libvirt/BUILD.bazel @@ -7,9 +7,9 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/file", - "@com_github_docker_docker//api/types", "@com_github_docker_docker//api/types/container", "@com_github_docker_docker//api/types/filters", + "@com_github_docker_docker//api/types/image", "@com_github_docker_docker//client", "@com_github_spf13_afero//:afero", ], diff --git a/cli/internal/libvirt/libvirt.go b/cli/internal/libvirt/libvirt.go index 1c8a584c7..5815ebfc5 100644 --- a/cli/internal/libvirt/libvirt.go +++ b/cli/internal/libvirt/libvirt.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -17,9 +17,9 @@ import ( "fmt" "io" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/image" docker "github.com/docker/docker/client" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/spf13/afero" @@ -55,7 +55,7 @@ func (r *Runner) Start(ctx context.Context, name, imageName string) error { // check for an existing container if containerName, err := r.file.Read(r.nameFile); err == nil { // check if a container with the same name already exists - containers, err := docker.ContainerList(ctx, types.ContainerListOptions{ + containers, err := docker.ContainerList(ctx, container.ListOptions{ Filters: filters.NewArgs( filters.KeyValuePair{ Key: "name", @@ -86,7 +86,7 @@ func (r *Runner) Start(ctx context.Context, name, imageName string) error { } // container exists but is not running, remove it // so we can start a new one - if err := docker.ContainerRemove(ctx, containers[0].ID, types.ContainerRemoveOptions{Force: true}); err != nil { + if err := docker.ContainerRemove(ctx, containers[0].ID, container.RemoveOptions{Force: true}); err != nil { return err } } @@ -101,7 +101,7 @@ func (r *Runner) Start(ctx context.Context, name, imageName string) error { func (r *Runner) startNewContainer(ctx context.Context, docker *docker.Client, containerName, imageName string) error { // check if image exists locally, if not pull it // this allows us to use a custom image without having to push it to a registry - images, err := docker.ImageList(ctx, types.ImageListOptions{ + images, err := docker.ImageList(ctx, image.ListOptions{ Filters: filters.NewArgs( filters.KeyValuePair{ Key: "reference", @@ -113,7 +113,7 @@ func (r *Runner) startNewContainer(ctx context.Context, docker *docker.Client, c return err } if len(images) == 0 { - reader, err := docker.ImagePull(ctx, imageName, types.ImagePullOptions{}) + reader, err := docker.ImagePull(ctx, imageName, image.PullOptions{}) if err != nil { return fmt.Errorf("failed to pull image %q: %w", imageName, err) } @@ -140,14 +140,14 @@ func (r *Runner) startNewContainer(ctx context.Context, docker *docker.Client, c ); err != nil { return fmt.Errorf("failed to create container: %w", err) } - if err := docker.ContainerStart(ctx, containerName, types.ContainerStartOptions{}); err != nil { - _ = docker.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{Force: true}) + if err := docker.ContainerStart(ctx, containerName, container.StartOptions{}); err != nil { + _ = docker.ContainerRemove(ctx, containerName, container.RemoveOptions{Force: true}) return fmt.Errorf("failed to start container: %w", err) } // write the name of the container to a file so we can remove it later if err := r.file.Write(r.nameFile, []byte(containerName)); err != nil { - _ = docker.ContainerRemove(ctx, containerName, types.ContainerRemoveOptions{Force: true}) + _ = docker.ContainerRemove(ctx, containerName, container.RemoveOptions{Force: true}) return err } @@ -169,7 +169,7 @@ func (r *Runner) Stop(ctx context.Context) error { return err } defer docker.Close() - if err := docker.ContainerRemove(ctx, string(name), types.ContainerRemoveOptions{Force: true}); err != nil { + if err := docker.ContainerRemove(ctx, string(name), container.RemoveOptions{Force: true}); err != nil { return err } diff --git a/cli/internal/terraform/BUILD.bazel b/cli/internal/terraform/BUILD.bazel index 919f87979..90ba2f86a 100644 --- a/cli/internal/terraform/BUILD.bazel +++ b/cli/internal/terraform/BUILD.bazel @@ -44,7 +44,7 @@ go_test( deps = [ "//internal/cloud/cloudprovider", "//internal/constants", - "//internal/constellation/state", + "//internal/encoding", "//internal/file", "//internal/role", "@com_github_azure_azure_sdk_for_go_sdk_azcore//to", diff --git a/cli/internal/terraform/loader.go b/cli/internal/terraform/loader.go index d6e448ce9..a3ad04482 100644 --- a/cli/internal/terraform/loader.go +++ b/cli/internal/terraform/loader.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package terraform diff --git a/cli/internal/terraform/loader_test.go b/cli/internal/terraform/loader_test.go index 4734bba1d..70a50240b 100644 --- a/cli/internal/terraform/loader_test.go +++ b/cli/internal/terraform/loader_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package terraform diff --git a/cli/internal/terraform/logging.go b/cli/internal/terraform/logging.go index 6a400fb03..18378d9f9 100644 --- a/cli/internal/terraform/logging.go +++ b/cli/internal/terraform/logging.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package terraform diff --git a/cli/internal/terraform/terraform.go b/cli/internal/terraform/terraform.go index 58ce818e1..c9b536109 100644 --- a/cli/internal/terraform/terraform.go +++ b/cli/internal/terraform/terraform.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -103,9 +103,18 @@ func (c *Client) ShowIAM(ctx context.Context, provider cloudprovider.Provider) ( if !ok { return IAMOutput{}, errors.New("invalid type in service_account_key output: not a string") } + IAMServiceAccountVMOutputRaw, ok := tfState.Values.Outputs["service_account_mail_vm"] + if !ok { + return IAMOutput{}, errors.New("no service_account_mail_vm output found") + } + IAMServiceAccountVMOutput, ok := IAMServiceAccountVMOutputRaw.Value.(string) + if !ok { + return IAMOutput{}, errors.New("invalid type in service_account_mail_vm output: not a string") + } return IAMOutput{ GCP: GCPIAMOutput{ - SaKey: saKeyOutput, + SaKey: saKeyOutput, + ServiceAccountVMMailAddress: IAMServiceAccountVMOutput, }, }, nil case cloudprovider.Azure: @@ -340,6 +349,28 @@ func (c *Client) ShowInfrastructure(ctx context.Context, provider cloudprovider. LoadBalancerName: loadBalancerName, AttestationURL: attestationURL, } + case cloudprovider.OpenStack: + networkIDOutput, ok := tfState.Values.Outputs["network_id"] + if !ok { + return state.Infrastructure{}, errors.New("no network_id output found") + } + networkID, ok := networkIDOutput.Value.(string) + if !ok { + return state.Infrastructure{}, errors.New("invalid type in network_id output: not a string") + } + lbSubnetworkIDOutput, ok := tfState.Values.Outputs["lb_subnetwork_id"] + if !ok { + return state.Infrastructure{}, errors.New("no lb_subnetwork_id output found") + } + lbSubnetworkID, ok := lbSubnetworkIDOutput.Value.(string) + if !ok { + return state.Infrastructure{}, errors.New("invalid type in lb_subnetwork_id output: not a string") + } + + res.OpenStack = &state.OpenStack{ + NetworkID: networkID, + SubnetID: lbSubnetworkID, + } } return res, nil } @@ -517,7 +548,8 @@ type IAMOutput struct { // GCPIAMOutput contains the output information of the Terraform IAM operation on GCP. type GCPIAMOutput struct { - SaKey string + SaKey string + ServiceAccountVMMailAddress string } // AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure. diff --git a/cli/internal/terraform/terraform_test.go b/cli/internal/terraform/terraform_test.go index d603679f3..07ea919e6 100644 --- a/cli/internal/terraform/terraform_test.go +++ b/cli/internal/terraform/terraform_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package terraform @@ -18,7 +18,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/constellation/state" + "github.com/edgelesssys/constellation/v2/internal/encoding" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/hashicorp/terraform-exec/tfexec" @@ -30,7 +30,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestPrepareCluster(t *testing.T) { @@ -120,6 +120,7 @@ func TestPrepareIAM(t *testing.T) { Region: "europe-west1", Zone: "europe-west1-a", ServiceAccountID: "const-test-case", + NamePrefix: "test_iam", } azureVars := &AzureIAMVariables{ Location: "westus", @@ -482,7 +483,7 @@ func TestCreateCluster(t *testing.T) { path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String())) require.NoError(c.PrepareWorkspace(path, tc.vars)) - infraState, err := c.ApplyCluster(context.Background(), tc.provider, LogLevelDebug) + infraState, err := c.ApplyCluster(t.Context(), tc.provider, LogLevelDebug) if tc.wantErr { assert.Error(err) @@ -490,7 +491,7 @@ func TestCreateCluster(t *testing.T) { } assert.NoError(err) assert.Equal("192.0.2.100", infraState.ClusterEndpoint) - assert.Equal(state.HexBytes("initSecret"), infraState.InitSecret) + assert.Equal(encoding.HexBytes("initSecret"), infraState.InitSecret) assert.Equal("12345abc", infraState.UID) assert.Equal("192.0.2.101", infraState.InClusterEndpoint) assert.Equal("192.0.2.103/32", infraState.IPCidrNode) @@ -509,6 +510,9 @@ func TestCreateIAM(t *testing.T) { "service_account_key": { Value: "12345678_abcdefg", }, + "service_account_mail_vm": { + Value: "test_iam_service_account_vm", + }, "subscription_id": { Value: "test_subscription_id", }, @@ -581,7 +585,7 @@ func TestCreateIAM(t *testing.T) { vars: gcpVars, tf: &stubTerraform{showState: newTestState()}, fs: afero.NewMemMapFs(), - want: IAMOutput{GCP: GCPIAMOutput{SaKey: "12345678_abcdefg"}}, + want: IAMOutput{GCP: GCPIAMOutput{SaKey: "12345678_abcdefg", ServiceAccountVMMailAddress: "test_iam_service_account_vm"}}, }, "gcp init fails": { pathBase: path.Join(constants.TerraformEmbeddedDir, "iam"), @@ -614,7 +618,25 @@ func TestCreateIAM(t *testing.T) { tf: &stubTerraform{ showState: &tfjson.State{ Values: &tfjson.StateValues{ - Outputs: map[string]*tfjson.StateOutput{}, + Outputs: map[string]*tfjson.StateOutput{ + "service_account_mail_vm": {Value: "test_iam_service_account_vm"}, + }, + }, + }, + }, + fs: afero.NewMemMapFs(), + wantErr: true, + }, + "gcp no service_account_mail_vm": { + pathBase: path.Join(constants.TerraformEmbeddedDir, "iam"), + provider: cloudprovider.GCP, + vars: gcpVars, + tf: &stubTerraform{ + showState: &tfjson.State{ + Values: &tfjson.StateValues{ + Outputs: map[string]*tfjson.StateOutput{ + "service_account_key": {Value: "12345678_abcdefg"}, + }, }, }, }, @@ -777,7 +799,7 @@ func TestCreateIAM(t *testing.T) { path := path.Join(tc.pathBase, strings.ToLower(tc.provider.String())) require.NoError(c.PrepareWorkspace(path, tc.vars)) - IAMoutput, err := c.ApplyIAM(context.Background(), tc.provider, LogLevelDebug) + IAMoutput, err := c.ApplyIAM(t.Context(), tc.provider, LogLevelDebug) if tc.wantErr { assert.Error(err) @@ -819,7 +841,7 @@ func TestDestroyInstances(t *testing.T) { tf: tc.tf, } - err := c.Destroy(context.Background(), LogLevelDebug) + err := c.Destroy(t.Context(), LogLevelDebug) if tc.wantErr { assert.Error(err) return @@ -852,7 +874,7 @@ func TestCleanupWorkspace(t *testing.T) { }, "no error if files do not exist": { provider: cloudprovider.QEMU, - prepareFS: func(f file.Handler) error { return nil }, + prepareFS: func(_ file.Handler) error { return nil }, }, } @@ -1051,7 +1073,7 @@ func TestPlan(t *testing.T) { workingDir: tc.pathBase, } - _, err := c.Plan(context.Background(), LogLevelDebug) + _, err := c.Plan(t.Context(), LogLevelDebug) if tc.wantErr { require.Error(err) } else { @@ -1110,7 +1132,7 @@ func TestShowPlan(t *testing.T) { workingDir: tc.pathBase, } - err := c.ShowPlan(context.Background(), LogLevelDebug, bytes.NewBuffer(nil)) + err := c.ShowPlan(t.Context(), LogLevelDebug, bytes.NewBuffer(nil)) if tc.wantErr { require.Error(err) } else { @@ -1129,7 +1151,8 @@ func TestShowIAM(t *testing.T) { "GCP success": { tf: &stubTerraform{ showState: getTfjsonState(map[string]any{ - "service_account_key": "key", + "service_account_key": "key", + "service_account_mail_vm": "example@example.com", }), }, csp: cloudprovider.GCP, @@ -1137,7 +1160,8 @@ func TestShowIAM(t *testing.T) { "GCP wrong data type": { tf: &stubTerraform{ showState: getTfjsonState(map[string]any{ - "service_account_key": map[string]any{}, + "service_account_key": map[string]any{}, + "service_account_mail_vm": "example@example.com", }), }, csp: cloudprovider.GCP, @@ -1145,7 +1169,9 @@ func TestShowIAM(t *testing.T) { }, "GCP missing key": { tf: &stubTerraform{ - showState: getTfjsonState(map[string]any{}), + showState: getTfjsonState(map[string]any{ + "service_account_mail_vm": "example@example.com", + }), }, csp: cloudprovider.GCP, wantErr: true, @@ -1294,7 +1320,7 @@ func TestShowIAM(t *testing.T) { tf: tc.tf, } - _, err := c.ShowIAM(context.Background(), tc.csp) + _, err := c.ShowIAM(t.Context(), tc.csp) if tc.wantErr { assert.Error(err) return diff --git a/cli/internal/terraform/variables.go b/cli/internal/terraform/variables.go index 6c800e124..d25b2e026 100644 --- a/cli/internal/terraform/variables.go +++ b/cli/internal/terraform/variables.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package terraform @@ -9,6 +9,7 @@ package terraform import ( "fmt" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -23,11 +24,6 @@ type Variables interface { // ClusterVariables should be used in places where a cluster is created. type ClusterVariables interface { Variables - // TODO(derpsteb): Rename this function once we have introduced an interface for config.Config. - // GetCreateMAA does not follow Go's naming convention because we need to keep the CreateMAA property public for now. - // There are functions creating Variables objects outside of this package. - // These functions can only be moved into this package once we have introduced an interface for config.Config, - // since we do not want to introduce a dependency on config.Config in this package. GetCreateMAA() bool } @@ -69,10 +65,11 @@ type AWSClusterVariables struct { CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"` // InternalLoadBalancer is true if an internal load balancer should be created. InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + // AdditionalTags describes (optional) additional tags that should be applied to created resources. + AdditionalTags cloudprovider.Tags `hcl:"additional_tags" cty:"additional_tags"` } // GetCreateMAA gets the CreateMAA variable. -// TODO(derpsteb): Rename this function once we have introduced an interface for config.Config. func (a *AWSClusterVariables) GetCreateMAA() bool { return false } @@ -136,10 +133,15 @@ type GCPClusterVariables struct { CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"` // InternalLoadBalancer is true if an internal load balancer should be created. InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + // CCTechnology is the confidential computing technology to use on the VMs. (`SEV` or `SEV_SNP`) + CCTechnology string `hcl:"cc_technology" cty:"cc_technology"` + // IAMServiceAccountControlPlane is the IAM service account mail address to attach to VMs. + IAMServiceAccountVM string `hcl:"iam_service_account_vm" cty:"iam_service_account_vm"` + // AdditionalLables are (optional) additional labels that should be applied to created resources. + AdditionalLabels cloudprovider.Tags `hcl:"additional_labels" cty:"additional_labels"` } // GetCreateMAA gets the CreateMAA variable. -// TODO(derpsteb): Rename this function once we have introduced an interface for config.Config. func (g *GCPClusterVariables) GetCreateMAA() bool { return false } @@ -165,7 +167,7 @@ type GCPNodeGroup struct { DiskType string `hcl:"disk_type" cty:"disk_type"` } -// GCPIAMVariables is user configuration for creating the IAM confioguration with Terraform on GCP. +// GCPIAMVariables is user configuration for creating the IAM configuration with Terraform on GCP. type GCPIAMVariables struct { // Project is the ID of the GCP project to use. Project string `hcl:"project_id" cty:"project_id"` @@ -175,6 +177,8 @@ type GCPIAMVariables struct { Zone string `hcl:"zone" cty:"zone"` // ServiceAccountID is the ID of the service account to use. ServiceAccountID string `hcl:"service_account_id" cty:"service_account_id"` + // NamePrefix is a prefix applied to the service account ID and VM ID created by this configuration. + NamePrefix string `hcl:"name_prefix,optional" cty:"name_prefix"` } // String returns a string representation of the IAM-specific variables, formatted as Terraform variables. @@ -186,6 +190,8 @@ func (v *GCPIAMVariables) String() string { // AzureClusterVariables is user configuration for creating a cluster with Terraform on Azure. type AzureClusterVariables struct { + // SubscriptionID is the Azure subscription ID to use. + SubscriptionID string `hcl:"subscription_id" cty:"subscription_id"` // Name of the cluster. Name string `hcl:"name" cty:"name"` // ImageID is the ID of the Azure image to use. @@ -212,10 +218,11 @@ type AzureClusterVariables struct { InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` // MarketplaceImage is the (optional) Azure Marketplace image to use. MarketplaceImage *AzureMarketplaceImageVariables `hcl:"marketplace_image" cty:"marketplace_image"` + // AdditionalTags are (optional) additional tags that get applied to created resources. + AdditionalTags cloudprovider.Tags `hcl:"additional_tags" cty:"additional_tags"` } // GetCreateMAA gets the CreateMAA variable. -// TODO(derpsteb): Rename this function once we have introduced an interface for config.Config. func (a *AzureClusterVariables) GetCreateMAA() bool { if a.CreateMAA == nil { return false @@ -245,11 +252,10 @@ type AzureNodeGroup struct { // AzureIAMVariables is user configuration for creating the IAM configuration with Terraform on Microsoft Azure. type AzureIAMVariables struct { - // Region is the Azure location to use. (e.g. westus). - // THIS FIELD IS DEPRECATED AND ONLY KEPT FOR MIGRATION PURPOSES. DO NOT USE. - Region *string `hcl:"region" cty:"region"` // TODO(msanft): Remove this field once v2.14.0 is released. + // SubscriptionID is the Azure subscription ID to use. + SubscriptionID string `hcl:"subscription_id,optional" cty:"subscription_id"` // TODO(v2.18): remove optional tag. This is only required for migration from var files that dont have the value yet. // Location is the Azure location to use. (e.g. westus) - Location string `hcl:"location,optional" cty:"location"` // TODO(msanft): Make this required once v2.14.0 is released. + Location string `hcl:"location" cty:"location"` // ServicePrincipal is the name of the service principal to use. ServicePrincipal string `hcl:"service_principal_name" cty:"service_principal_name"` // ResourceGroup is the name of the resource group to use. @@ -281,30 +287,26 @@ type OpenStackClusterVariables struct { Name string `hcl:"name" cty:"name"` // NodeGroups is a map of node groups to create. NodeGroups map[string]OpenStackNodeGroup `hcl:"node_groups" cty:"node_groups"` - // Cloud is the (optional) name of the OpenStack cloud to use when reading the "clouds.yaml" configuration file. If empty, environment variables are used. + // Cloud is the name of the OpenStack cloud to use when reading the "clouds.yaml" configuration file. If empty, environment variables are used. Cloud *string `hcl:"cloud" cty:"cloud"` + // OpenStackCloudsYAMLPath is the path to the OpenStack clouds.yaml file + OpenStackCloudsYAMLPath string `hcl:"openstack_clouds_yaml_path" cty:"openstack_clouds_yaml_path"` + // (STACKIT only) STACKITProjectID is the ID of the STACKIT project to use. + STACKITProjectID string `hcl:"stackit_project_id" cty:"stackit_project_id"` // FloatingIPPoolID is the ID of the OpenStack floating IP pool to use for public IPs. FloatingIPPoolID string `hcl:"floating_ip_pool_id" cty:"floating_ip_pool_id"` - // ImageURL is the URL of the OpenStack image to use. - ImageURL string `hcl:"image_id" cty:"image_id"` - // DirectDownload decides whether to download the image directly from the URL to OpenStack or to upload it from the local machine. - DirectDownload bool `hcl:"direct_download" cty:"direct_download"` - // OpenstackUserDomainName is the OpenStack user domain name to use. - OpenstackUserDomainName string `hcl:"openstack_user_domain_name" cty:"openstack_user_domain_name"` - // OpenstackUsername is the OpenStack user name to use. - OpenstackUsername string `hcl:"openstack_username" cty:"openstack_username"` - // OpenstackPassword is the OpenStack password to use. - OpenstackPassword string `hcl:"openstack_password" cty:"openstack_password"` + // ImageID is the ID of the OpenStack image to use. + ImageID string `hcl:"image_id" cty:"image_id"` // Debug is true if debug mode is enabled. Debug bool `hcl:"debug" cty:"debug"` // CustomEndpoint is the (optional) custom dns hostname for the kubernetes api server. CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"` // InternalLoadBalancer is true if an internal load balancer should be created. - InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"` + AdditionalTags []string `hcl:"additional_tags" cty:"additional_tags"` } // GetCreateMAA gets the CreateMAA variable. -// TODO(derpsteb): Rename this function once we have introduced an interface for config.Config. func (o *OpenStackClusterVariables) GetCreateMAA() bool { return false } @@ -377,7 +379,6 @@ type QEMUVariables struct { } // GetCreateMAA gets the CreateMAA variable. -// TODO(derpsteb): Rename this function once we have introduced an interface for config.Config. func (q *QEMUVariables) GetCreateMAA() bool { return false } diff --git a/cli/internal/terraform/variables_test.go b/cli/internal/terraform/variables_test.go index 30ad43e6b..dc8f79b2d 100644 --- a/cli/internal/terraform/variables_test.go +++ b/cli/internal/terraform/variables_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package terraform @@ -76,6 +76,7 @@ node_groups = { } custom_endpoint = "example.com" internal_load_balancer = false +additional_tags = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences @@ -121,7 +122,9 @@ func TestGCPClusterVariables(t *testing.T) { DiskType: "pd-ssd", }, }, - CustomEndpoint: "example.com", + CustomEndpoint: "example.com", + CCTechnology: "SEV_SNP", + IAMServiceAccountVM: "example@example.com", } // test that the variables are correctly rendered @@ -149,8 +152,11 @@ node_groups = { zone = "eu-central-1b" } } -custom_endpoint = "example.com" -internal_load_balancer = false +custom_endpoint = "example.com" +internal_load_balancer = false +cc_technology = "SEV_SNP" +iam_service_account_vm = "example@example.com" +additional_labels = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences @@ -169,14 +175,33 @@ func TestGCPIAMVariables(t *testing.T) { region = "eu-central-1" zone = "eu-central-1a" service_account_id = "my-service-account" +name_prefix = "" ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences + + vars = GCPIAMVariables{ + Project: "my-project", + Region: "eu-central-1", + Zone: "eu-central-1a", + NamePrefix: "my-prefix", + } + + // test that the variables are correctly rendered + want = `project_id = "my-project" +region = "eu-central-1" +zone = "eu-central-1a" +service_account_id = "" +name_prefix = "my-prefix" +` + got = vars.String() + assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences } func TestAzureClusterVariables(t *testing.T) { vars := AzureClusterVariables{ - Name: "cluster-name", + SubscriptionID: "01234567-cdef-0123-4567-89abcdef0123", + Name: "cluster-name", NodeGroups: map[string]AzureNodeGroup{ constants.ControlPlaneDefault: { Role: "ControlPlane", @@ -203,7 +228,8 @@ func TestAzureClusterVariables(t *testing.T) { } // test that the variables are correctly rendered - want := `name = "cluster-name" + want := `subscription_id = "01234567-cdef-0123-4567-89abcdef0123" +name = "cluster-name" image_id = "image-0123456789abcdef" create_maa = true debug = true @@ -229,6 +255,7 @@ marketplace_image = { publisher = "edgelesssys" version = "2.13.0" } +additional_tags = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences @@ -236,13 +263,15 @@ marketplace_image = { func TestAzureIAMVariables(t *testing.T) { vars := AzureIAMVariables{ + SubscriptionID: "01234567-cdef-0123-4567-89abcdef0123", Location: "eu-central-1", ServicePrincipal: "my-service-principal", ResourceGroup: "my-resource-group", } // test that the variables are correctly rendered - want := `location = "eu-central-1" + want := `subscription_id = "01234567-cdef-0123-4567-89abcdef0123" +location = "eu-central-1" service_principal_name = "my-service-principal" resource_group_name = "my-resource-group" ` @@ -254,13 +283,11 @@ func TestOpenStackClusterVariables(t *testing.T) { vars := OpenStackClusterVariables{ Name: "cluster-name", Cloud: toPtr("my-cloud"), + OpenStackCloudsYAMLPath: "~/.config/openstack/clouds.yaml", FloatingIPPoolID: "fip-pool-0123456789abcdef", - ImageURL: "https://example.com/image.raw", - DirectDownload: true, - OpenstackUserDomainName: "my-user-domain", - OpenstackUsername: "my-username", - OpenstackPassword: "my-password", + ImageID: "8e10b92d-8f7a-458c-91c6-59b42f82ef81", Debug: true, + STACKITProjectID: "my-stackit-project-id", NodeGroups: map[string]OpenStackNodeGroup{ constants.ControlPlaneDefault: { Role: "control-plane", @@ -287,15 +314,14 @@ node_groups = { } } cloud = "my-cloud" +openstack_clouds_yaml_path = "~/.config/openstack/clouds.yaml" +stackit_project_id = "my-stackit-project-id" floating_ip_pool_id = "fip-pool-0123456789abcdef" -image_id = "https://example.com/image.raw" -direct_download = true -openstack_user_domain_name = "my-user-domain" -openstack_username = "my-username" -openstack_password = "my-password" +image_id = "8e10b92d-8f7a-458c-91c6-59b42f82ef81" debug = true custom_endpoint = "example.com" internal_load_balancer = false +additional_tags = null ` got := vars.String() assert.Equal(t, strings.Fields(want), strings.Fields(got)) // to ignore whitespace differences diff --git a/cli/main.go b/cli/main.go index 7687463bd..0d479766d 100644 --- a/cli/main.go +++ b/cli/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main diff --git a/csi/cryptmapper/cryptmapper.go b/csi/cryptmapper/cryptmapper.go index 90ece1df2..44757d703 100644 --- a/csi/cryptmapper/cryptmapper.go +++ b/csi/cryptmapper/cryptmapper.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package cryptmapper provides a wrapper around libcryptsetup to manage dm-crypt volumes for CSI drivers. diff --git a/csi/cryptmapper/cryptmapper_cgo.go b/csi/cryptmapper/cryptmapper_cgo.go index f03a48bbb..0ba881d6d 100644 --- a/csi/cryptmapper/cryptmapper_cgo.go +++ b/csi/cryptmapper/cryptmapper_cgo.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cryptmapper diff --git a/csi/cryptmapper/cryptmapper_cross.go b/csi/cryptmapper/cryptmapper_cross.go index ddc4f4adc..0e6f6ba34 100644 --- a/csi/cryptmapper/cryptmapper_cross.go +++ b/csi/cryptmapper/cryptmapper_cross.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cryptmapper diff --git a/csi/cryptmapper/cryptmapper_test.go b/csi/cryptmapper/cryptmapper_test.go index fbc9dcd63..56c44e8ee 100644 --- a/csi/cryptmapper/cryptmapper_test.go +++ b/csi/cryptmapper/cryptmapper_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cryptmapper @@ -18,7 +18,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestCloseCryptDevice(t *testing.T) { @@ -81,7 +81,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: false, }, "success with error on Load": { @@ -89,7 +89,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: false, }, "success with integrity": { @@ -98,7 +98,7 @@ func TestOpenCryptDevice(t *testing.T) { integrity: true, mapper: &stubCryptDevice{loadErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: false, }, "error on Init": { @@ -106,7 +106,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{initErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "error on Format": { @@ -114,7 +114,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError, formatErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "error on Activate": { @@ -122,7 +122,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{activatePassErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "error on diskInfo": { @@ -130,7 +130,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", assert.AnError }, + diskInfo: func(_ string) (string, error) { return "", assert.AnError }, wantErr: true, }, "disk is already formatted": { @@ -138,7 +138,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "ext4", nil }, + diskInfo: func(_ string) (string, error) { return "ext4", nil }, wantErr: true, }, "error with integrity on wipe": { @@ -147,7 +147,7 @@ func TestOpenCryptDevice(t *testing.T) { integrity: true, mapper: &stubCryptDevice{loadErr: assert.AnError, wipeErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "error on adding keyslot": { @@ -155,7 +155,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError, keySlotAddErr: assert.AnError}, kms: &fakeKMS{}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "incorrect key length": { @@ -163,7 +163,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{}, kms: &fakeKMS{presetKey: []byte{0x1, 0x2, 0x3}}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "incorrect key length with error on Load": { @@ -171,7 +171,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError}, kms: &fakeKMS{presetKey: []byte{0x1, 0x2, 0x3}}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "getKey fails": { @@ -179,7 +179,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{}, kms: &fakeKMS{getDEKErr: assert.AnError}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, "getKey fails with error on Load": { @@ -187,7 +187,7 @@ func TestOpenCryptDevice(t *testing.T) { volumeID: "volume0", mapper: &stubCryptDevice{loadErr: assert.AnError}, kms: &fakeKMS{getDEKErr: assert.AnError}, - diskInfo: func(disk string) (string, error) { return "", nil }, + diskInfo: func(_ string) (string, error) { return "", nil }, wantErr: true, }, } @@ -202,7 +202,7 @@ func TestOpenCryptDevice(t *testing.T) { getDiskFormat: tc.diskInfo, } - out, err := mapper.OpenCryptDevice(context.Background(), tc.source, tc.volumeID, tc.integrity) + out, err := mapper.OpenCryptDevice(t.Context(), tc.source, tc.volumeID, tc.integrity) if tc.wantErr { assert.Error(err) } else { @@ -223,7 +223,7 @@ func TestOpenCryptDevice(t *testing.T) { kms: &fakeKMS{}, getDiskFormat: getDiskFormat, } - _, err := mapper.OpenCryptDevice(context.Background(), "/dev/some-device", "volume01", false) + _, err := mapper.OpenCryptDevice(t.Context(), "/dev/some-device", "volume01", false) assert.NoError(t, err) } @@ -270,7 +270,7 @@ func TestResizeCryptDevice(t *testing.T) { mapper: testMapper(tc.device), } - res, err := mapper.ResizeCryptDevice(context.Background(), tc.volumeID) + res, err := mapper.ResizeCryptDevice(t.Context(), tc.volumeID) if tc.wantErr { assert.Error(err) } else { diff --git a/csi/kms/BUILD.bazel b/csi/kms/BUILD.bazel index 19d174a01..81dd9ecd8 100644 --- a/csi/kms/BUILD.bazel +++ b/csi/kms/BUILD.bazel @@ -8,7 +8,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//keyservice/keyserviceproto", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//credentials/insecure", ], ) @@ -20,7 +20,7 @@ go_test( deps = [ "//keyservice/keyserviceproto", "@com_github_stretchr_testify//assert", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//test/bufconn", "@org_uber_go_goleak//:goleak", ], diff --git a/csi/kms/constellation.go b/csi/kms/constellation.go index dbc310bdb..95e4f848e 100644 --- a/csi/kms/constellation.go +++ b/csi/kms/constellation.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kms @@ -31,7 +31,7 @@ func NewConstellationKMS(endpoint string) *ConstellationKMS { // GetDEK request a data encryption key derived from the Constellation's master secret. func (k *ConstellationKMS) GetDEK(ctx context.Context, dekID string, dekSize int) ([]byte, error) { - conn, err := grpc.DialContext(ctx, k.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(k.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, err } diff --git a/csi/kms/constellation_test.go b/csi/kms/constellation_test.go index 4b362d287..134404491 100644 --- a/csi/kms/constellation_test.go +++ b/csi/kms/constellation_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kms @@ -19,7 +19,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } type stubKMSClient struct { @@ -57,7 +57,7 @@ func TestConstellationKMS(t *testing.T) { endpoint: listener.Addr().String(), kms: tc.kms, } - res, err := kms.GetDEK(context.Background(), "data-key", 64) + res, err := kms.GetDEK(t.Context(), "data-key", 64) if tc.wantErr { assert.Error(err) diff --git a/csi/test/BUILD.bazel b/csi/test/BUILD.bazel index b62498c20..c2c5b1071 100644 --- a/csi/test/BUILD.bazel +++ b/csi/test/BUILD.bazel @@ -28,6 +28,8 @@ go_test( "RM": "$(rlocationpath @coreutils//:bin/rm)", "UMOUNT": "$(rlocationpath @util-linux//:bin/umount)", }, + # This test frequently runs into https://github.com/martinjungblut/go-cryptsetup/issues/13. + flaky = 1, # keep tags = [ "integration", @@ -40,14 +42,14 @@ go_test( "//csi/cryptmapper", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", + "@io_bazel_rules_go//go/runfiles", "@org_uber_go_goleak//:goleak", ], "@io_bazel_rules_go//go/platform:linux": [ "//csi/cryptmapper", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", + "@io_bazel_rules_go//go/runfiles", "@org_uber_go_goleak//:goleak", ], "//conditions:default": [], diff --git a/csi/test/mount_integration_test.go b/csi/test/mount_integration_test.go index 6b01a0a62..c22371c2e 100644 --- a/csi/test/mount_integration_test.go +++ b/csi/test/mount_integration_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package integration @@ -31,7 +31,7 @@ const ( deviceName string = "testDeviceName" ) -var toolsEnvs []string = []string{"CP", "DD", "RM", "FSCK_EXT4", "MKFS_EXT4", "BLKID", "FSCK", "MOUNT", "UMOUNT"} +var toolsEnvs = []string{"CP", "DD", "RM", "FSCK_EXT4", "MKFS_EXT4", "BLKID", "FSCK", "MOUNT", "UMOUNT"} // addToolsToPATH is used to update the PATH to contain necessary tool binaries for // coreutils, util-linux and ext4. @@ -91,7 +91,7 @@ func TestMain(m *testing.M) { os.Exit(1) } - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) result := m.Run() os.Exit(result) @@ -105,7 +105,7 @@ func TestOpenAndClose(t *testing.T) { mapper := cryptmapper.New(&fakeKMS{}) - newPath, err := mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, false) + newPath, err := mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, false) require.NoError(err) defer func() { _ = mapper.CloseCryptDevice(deviceName) @@ -119,14 +119,14 @@ func TestOpenAndClose(t *testing.T) { assert.True(os.IsNotExist(err)) // Opening the same device should return the same path and not error - newPath2, err := mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, false) + newPath2, err := mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, false) require.NoError(err) assert.Equal(newPath, newPath2) // Resize the device resize(devicePath) - resizedPath, err := mapper.ResizeCryptDevice(context.Background(), deviceName) + resizedPath, err := mapper.ResizeCryptDevice(t.Context(), deviceName) require.NoError(err) assert.Equal("/dev/mapper/"+deviceName, resizedPath) @@ -137,7 +137,7 @@ func TestOpenAndClose(t *testing.T) { assert.True(os.IsNotExist(err)) // check if we can reopen the device - _, err = mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, true) + _, err = mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, true) assert.NoError(err) assert.NoError(mapper.CloseCryptDevice(deviceName)) } @@ -150,7 +150,7 @@ func TestOpenAndCloseIntegrity(t *testing.T) { mapper := cryptmapper.New(&fakeKMS{}) - newPath, err := mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, true) + newPath, err := mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, true) require.NoError(err) assert.Equal("/dev/mapper/"+deviceName, newPath) @@ -162,13 +162,13 @@ func TestOpenAndCloseIntegrity(t *testing.T) { assert.NoError(err) // Opening the same device should return the same path and not error - newPath2, err := mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, true) + newPath2, err := mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, true) require.NoError(err) assert.Equal(newPath, newPath2) // integrity devices do not support resizing resize(devicePath) - _, err = mapper.ResizeCryptDevice(context.Background(), deviceName) + _, err = mapper.ResizeCryptDevice(t.Context(), deviceName) assert.Error(err) assert.NoError(mapper.CloseCryptDevice(deviceName)) @@ -181,7 +181,7 @@ func TestOpenAndCloseIntegrity(t *testing.T) { assert.True(os.IsNotExist(err)) // check if we can reopen the device - _, err = mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, true) + _, err = mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, true) assert.NoError(err) assert.NoError(mapper.CloseCryptDevice(deviceName)) } @@ -194,13 +194,13 @@ func TestDeviceCloning(t *testing.T) { mapper := cryptmapper.New(&dynamicKMS{}) - _, err := mapper.OpenCryptDevice(context.Background(), devicePath, deviceName, false) + _, err := mapper.OpenCryptDevice(t.Context(), devicePath, deviceName, false) assert.NoError(err) require.NoError(cp(devicePath, devicePath+"-copy")) defer teardown(devicePath + "-copy") - _, err = mapper.OpenCryptDevice(context.Background(), devicePath+"-copy", deviceName+"-copy", false) + _, err = mapper.OpenCryptDevice(t.Context(), devicePath+"-copy", deviceName+"-copy", false) assert.NoError(err) assert.NoError(mapper.CloseCryptDevice(deviceName)) @@ -220,7 +220,7 @@ func TestConcurrency(t *testing.T) { wg := sync.WaitGroup{} runTest := func(path, name string) { - newPath, err := mapper.OpenCryptDevice(context.Background(), path, name, false) + newPath, err := mapper.OpenCryptDevice(t.Context(), path, name, false) assert.NoError(err) defer func() { _ = mapper.CloseCryptDevice(name) diff --git a/debugd/cmd/cdbg/cdbg.go b/debugd/cmd/cdbg/cdbg.go index 24d00f21f..d7962a8c4 100644 --- a/debugd/cmd/cdbg/cdbg.go +++ b/debugd/cmd/cdbg/cdbg.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main diff --git a/debugd/cmd/debugd/BUILD.bazel b/debugd/cmd/debugd/BUILD.bazel index 5f4e716e4..e517ef6a9 100644 --- a/debugd/cmd/debugd/BUILD.bazel +++ b/debugd/cmd/debugd/BUILD.bazel @@ -25,7 +25,6 @@ go_library( "//internal/cloud/qemu", "//internal/logger", "@com_github_spf13_afero//:afero", - "@org_uber_go_zap//:zap", ], ) diff --git a/debugd/cmd/debugd/debugd.go b/debugd/cmd/debugd/debugd.go index 92f7f1614..02ebfc96e 100644 --- a/debugd/cmd/debugd/debugd.go +++ b/debugd/cmd/debugd/debugd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -10,12 +10,12 @@ import ( "context" "flag" "fmt" + "log/slog" "net" "os" "sync" "github.com/spf13/afero" - "go.uber.org/zap" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/deploy" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/info" @@ -46,11 +46,11 @@ func main() { verbosity := flag.Int("v", 0, logger.CmdLineVerbosityDescription) flag.Parse() - log := logger.New(logger.JSONLog, logger.VerbosityFromInt(*verbosity)) + log := logger.NewJSONLogger(logger.VerbosityFromInt(*verbosity)) fs := afero.NewOsFs() streamer := streamer.New(fs) - filetransferer := filetransfer.New(log.Named("filetransfer"), streamer, filetransfer.DontShowProgress) - serviceManager := deploy.NewServiceManager(log.Named("serviceManager")) + filetransferer := filetransfer.New(log.WithGroup("filetransfer"), streamer, filetransfer.DontShowProgress) + serviceManager := deploy.NewServiceManager(log.WithGroup("serviceManager")) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -64,21 +64,24 @@ func main() { case platform.AWS: meta, err := awscloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to initialize AWS metadata") + log.With(slog.Any("error", err)).Error("Failed to initialize AWS metadata") + os.Exit(1) } fetcher = cloudprovider.New(meta) case platform.Azure: meta, err := azurecloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to initialize Azure metadata") + log.With(slog.Any("error", err)).Error("Failed to initialize Azure metadata") + os.Exit(1) } fetcher = cloudprovider.New(meta) case platform.GCP: meta, err := gcpcloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to initialize GCP metadata") + log.With(slog.Any("error", err)).Error("Failed to initialize GCP metadata") + os.Exit(1) } defer meta.Close() fetcher = cloudprovider.New(meta) @@ -86,26 +89,27 @@ func main() { case platform.OpenStack: meta, err := openstackcloud.New(ctx) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to initialize OpenStack metadata") + log.With(slog.Any("error", err)).Error("Failed to initialize OpenStack metadata") + os.Exit(1) } fetcher = cloudprovider.New(meta) case platform.QEMU: fetcher = cloudprovider.New(qemucloud.New()) default: - log.Errorf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", csp) + log.Error(fmt.Sprintf("Unknown / unimplemented cloud provider CONSTEL_CSP=%v. Using fallback", csp)) fetcher = fallback.NewFallbackFetcher() } infoMap := info.NewMap() infoMap.RegisterOnReceiveTrigger( - logcollector.NewStartTrigger(ctx, wg, platform.FromString(csp), fetcher, log.Named("logcollector")), + logcollector.NewStartTrigger(ctx, wg, platform.FromString(csp), fetcher, log.WithGroup("logcollector")), ) - download := deploy.New(log.Named("download"), &net.Dialer{}, serviceManager, filetransferer, infoMap) + download := deploy.New(log.WithGroup("download"), &net.Dialer{}, serviceManager, filetransferer, infoMap) - sched := metadata.NewScheduler(log.Named("scheduler"), fetcher, download) - serv := server.New(log.Named("server"), serviceManager, filetransferer, infoMap) + sched := metadata.NewScheduler(log.WithGroup("scheduler"), fetcher, download) + serv := server.New(log.WithGroup("server"), serviceManager, filetransferer, infoMap) writeDebugBanner(log) @@ -114,14 +118,14 @@ func main() { wg.Wait() } -func writeDebugBanner(log *logger.Logger) { +func writeDebugBanner(log *slog.Logger) { tty, err := os.OpenFile("/dev/ttyS0", os.O_WRONLY, os.ModeAppend) if err != nil { - log.With(zap.Error(err)).Errorf("Unable to open /dev/ttyS0 for printing banner") + log.With(slog.Any("error", err)).Error("Unable to open /dev/ttyS0 for printing banner") return } defer tty.Close() if _, err := fmt.Fprint(tty, debugBanner); err != nil { - log.With(zap.Error(err)).Errorf("Unable to print to /dev/ttyS0") + log.With(slog.Any("error", err)).Error("Unable to print to /dev/ttyS0") } } diff --git a/debugd/filebeat/Dockerfile b/debugd/filebeat/Dockerfile index 58ba79e81..f5badcf9a 100644 --- a/debugd/filebeat/Dockerfile +++ b/debugd/filebeat/Dockerfile @@ -1,7 +1,9 @@ -FROM fedora:38@sha256:8285246bd5fad4e76e17a71c88dee34c49e2f227dab4ce7df704b592f8e72d41 AS release +FROM fedora:40@sha256:3c86d25fef9d2001712bc3d9b091fc40cf04be4767e48f1aa3b785bf58d300ed AS release RUN dnf install -y https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.6.2-x86_64.rpm +RUN dnf install -y systemd-libs + COPY debugd/filebeat/templates/ /usr/share/constellogs/templates/ ENTRYPOINT ["/usr/share/filebeat/bin/filebeat", "-e", "--path.home", "/usr/share/filebeat", "--path.data", "/usr/share/filebeat/data"] diff --git a/debugd/filebeat/assets.go b/debugd/filebeat/assets.go index 744ef3799..204b1a3ec 100644 --- a/debugd/filebeat/assets.go +++ b/debugd/filebeat/assets.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package filebeat diff --git a/debugd/internal/cdbg/cmd/BUILD.bazel b/debugd/internal/cdbg/cmd/BUILD.bazel index 94420b1c1..e48d0d5d1 100644 --- a/debugd/internal/cdbg/cmd/BUILD.bazel +++ b/debugd/internal/cdbg/cmd/BUILD.bazel @@ -22,7 +22,7 @@ go_library( "//internal/logger", "@com_github_spf13_afero//:afero", "@com_github_spf13_cobra//:cobra", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//credentials/insecure", ], ) diff --git a/debugd/internal/cdbg/cmd/deploy.go b/debugd/internal/cdbg/cmd/deploy.go index 9705df4a1..ea7569d28 100644 --- a/debugd/internal/cdbg/cmd/deploy.go +++ b/debugd/internal/cdbg/cmd/deploy.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "path/filepath" "strconv" @@ -60,7 +61,7 @@ func runDeploy(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, logger.VerbosityFromInt(verbosity)) + log := logger.NewTextLogger(logger.VerbosityFromInt(verbosity)) force, err := cmd.Flags().GetBool("force") if err != nil { return fmt.Errorf("getting force flag: %w", err) @@ -83,7 +84,7 @@ func runDeploy(cmd *cobra.Command, _ []string) error { func deploy(cmd *cobra.Command, fileHandler file.Handler, constellationConfig *config.Config, transfer fileTransferer, - log *logger.Logger, + log *slog.Logger, ) error { binDir, err := cmd.Flags().GetString("bindir") if err != nil { @@ -99,13 +100,13 @@ func deploy(cmd *cobra.Command, fileHandler file.Handler, constellationConfig *c } if constellationConfig.IsReleaseImage() { - log.Infof("WARNING: Constellation image does not look like a debug image. Are you using a debug image?") + log.Info("WARNING: Constellation image does not look like a debug image. Are you using a debug image?") } if !constellationConfig.IsDebugCluster() { - log.Infof("WARNING: The Constellation config has debugCluster set to false.") - log.Infof("cdbg will likely not work unless you manually adjust the firewall / load balancing rules.") - log.Infof("If you create the cluster with a debug image, you should also set debugCluster to true.") + log.Info("WARNING: The Constellation config has debugCluster set to false.") + log.Info("cdbg will likely not work unless you manually adjust the firewall / load balancing rules.") + log.Info("If you create the cluster with a debug image, you should also set debugCluster to true.") } ips, err := cmd.Flags().GetStringSlice("ips") @@ -171,14 +172,14 @@ type deployOnEndpointInput struct { files []filetransfer.FileStat infos map[string]string transfer fileTransferer - log *logger.Logger + log *slog.Logger } // deployOnEndpoint deploys a custom built bootstrapper binary to a debugd endpoint. func deployOnEndpoint(ctx context.Context, in deployOnEndpointInput) error { ctx, cancel := context.WithTimeout(ctx, deployEndpointTimeout) defer cancel() - in.log.Infof("Deploying on %v", in.debugdEndpoint) + in.log.Info(fmt.Sprintf("Deploying on %v", in.debugdEndpoint)) client, closeAndWaitFn, err := newDebugdClient(ctx, in.debugdEndpoint, in.log) if err != nil { @@ -201,13 +202,12 @@ func deployOnEndpoint(ctx context.Context, in deployOnEndpointInput) error { type closeAndWait func() // newDebugdClient creates a new gRPC client for the debugd service and logs the connection state changes. -func newDebugdClient(ctx context.Context, ip string, log *logger.Logger) (pb.DebugdClient, closeAndWait, error) { - conn, err := grpc.DialContext( - ctx, +func newDebugdClient(ctx context.Context, ip string, log *slog.Logger) (pb.DebugdClient, closeAndWait, error) { + conn, err := grpc.NewClient( net.JoinHostPort(ip, strconv.Itoa(constants.DebugdPort)), grpc.WithTransportCredentials(insecure.NewCredentials()), - log.GetClientUnaryInterceptor(), - log.GetClientStreamInterceptor(), + logger.GetClientUnaryInterceptor(log), + logger.GetClientStreamInterceptor(log), ) if err != nil { return nil, nil, fmt.Errorf("connecting to other instance via gRPC: %w", err) @@ -221,8 +221,8 @@ func newDebugdClient(ctx context.Context, ip string, log *logger.Logger) (pb.Deb return pb.NewDebugdClient(conn), closeAndWait, nil } -func setInfo(ctx context.Context, log *logger.Logger, client pb.DebugdClient, infos map[string]string) error { - log.Infof("Setting info with length %d", len(infos)) +func setInfo(ctx context.Context, log *slog.Logger, client pb.DebugdClient, infos map[string]string) error { + log.Info(fmt.Sprintf("Setting info with length %d", len(infos))) var infosPb []*pb.Info for key, value := range infos { @@ -238,17 +238,17 @@ func setInfo(ctx context.Context, log *logger.Logger, client pb.DebugdClient, in switch status.Status { case pb.SetInfoStatus_SET_INFO_SUCCESS: - log.Infof("Info set") + log.Info("Info set") case pb.SetInfoStatus_SET_INFO_ALREADY_SET: - log.Infof("Info already set") + log.Info("Info already set") default: - log.Warnf("Unknown status %v", status.Status) + log.Warn(fmt.Sprintf("Unknown status %v", status.Status)) } return nil } func uploadFiles(ctx context.Context, client pb.DebugdClient, in deployOnEndpointInput) error { - in.log.Infof("Uploading files") + in.log.Info("Uploading files") stream, err := client.UploadFiles(ctx, grpc.WaitForReady(true)) if err != nil { @@ -266,13 +266,15 @@ func uploadFiles(ctx context.Context, client pb.DebugdClient, in deployOnEndpoin } switch uploadResponse.Status { case pb.UploadFilesStatus_UPLOAD_FILES_SUCCESS: - in.log.Infof("Upload successful") + in.log.Info("Upload successful") case pb.UploadFilesStatus_UPLOAD_FILES_ALREADY_FINISHED: - in.log.Infof("Files already uploaded") + in.log.Info("Files already uploaded") case pb.UploadFilesStatus_UPLOAD_FILES_UPLOAD_FAILED: - return fmt.Errorf("uploading files to %v failed: %v", in.debugdEndpoint, uploadResponse) + return fmt.Errorf("uploading files to %v failed: %s: %s", in.debugdEndpoint, uploadResponse.Status, uploadResponse.Error) case pb.UploadFilesStatus_UPLOAD_FILES_ALREADY_STARTED: return fmt.Errorf("upload already started on %v", in.debugdEndpoint) + case pb.UploadFilesStatus_UPLOAD_FILES_START_FAILED: + return fmt.Errorf("overriding service units failed on %v: %s: %s", in.debugdEndpoint, uploadResponse.Status, uploadResponse.Error) default: return fmt.Errorf("unknown upload status %v", uploadResponse.Status) } diff --git a/debugd/internal/cdbg/cmd/root.go b/debugd/internal/cdbg/cmd/root.go index b9b3fae67..436b524a8 100644 --- a/debugd/internal/cdbg/cmd/root.go +++ b/debugd/internal/cdbg/cmd/root.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package cmd contains the cdbg CLI. diff --git a/debugd/internal/debugd/constants.go b/debugd/internal/debugd/constants.go index ae3aab8b3..e831fc8cb 100644 --- a/debugd/internal/debugd/constants.go +++ b/debugd/internal/debugd/constants.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package debugd diff --git a/debugd/internal/debugd/debugd.go b/debugd/internal/debugd/debugd.go index d5453dbf8..645447c0a 100644 --- a/debugd/internal/debugd/debugd.go +++ b/debugd/internal/debugd/debugd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package debugd contains internal packages for the debugd. diff --git a/debugd/internal/debugd/deploy/BUILD.bazel b/debugd/internal/debugd/deploy/BUILD.bazel index 4e0cccb4f..29bdd0ccf 100644 --- a/debugd/internal/debugd/deploy/BUILD.bazel +++ b/debugd/internal/debugd/deploy/BUILD.bazel @@ -16,12 +16,10 @@ go_library( "//debugd/internal/filetransfer", "//debugd/service", "//internal/constants", - "//internal/logger", "@com_github_coreos_go_systemd_v22//dbus", "@com_github_spf13_afero//:afero", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//credentials/insecure", - "@org_uber_go_zap//:zap", ], ) @@ -41,7 +39,7 @@ go_test( "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_uber_go_goleak//:goleak", ], ) diff --git a/debugd/internal/debugd/deploy/deploy.go b/debugd/internal/debugd/deploy/deploy.go index d91e0243f..7e43f70cd 100644 --- a/debugd/internal/debugd/deploy/deploy.go +++ b/debugd/internal/debugd/deploy/deploy.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/debugd/internal/debugd/deploy/download.go b/debugd/internal/debugd/deploy/download.go index b4000ff2c..0409389f3 100644 --- a/debugd/internal/debugd/deploy/download.go +++ b/debugd/internal/debugd/deploy/download.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package deploy @@ -11,21 +11,20 @@ import ( "errors" "fmt" "io" + "log/slog" "net" "strconv" "github.com/edgelesssys/constellation/v2/debugd/internal/filetransfer" pb "github.com/edgelesssys/constellation/v2/debugd/service" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) // Download downloads a bootstrapper from a given debugd instance. type Download struct { - log *logger.Logger + log *slog.Logger dialer NetDialer transfer fileTransferer serviceManager serviceManager @@ -33,7 +32,7 @@ type Download struct { } // New creates a new Download. -func New(log *logger.Logger, dialer NetDialer, serviceManager serviceManager, +func New(log *slog.Logger, dialer NetDialer, serviceManager serviceManager, transfer fileTransferer, info infoSetter, ) *Download { return &Download{ @@ -51,37 +50,37 @@ func (d *Download) DownloadInfo(ctx context.Context, ip string) error { return nil } - log := d.log.With(zap.String("ip", ip)) + log := d.log.With(slog.String("ip", ip)) serverAddr := net.JoinHostPort(ip, strconv.Itoa(constants.DebugdPort)) - client, closer, err := d.newClient(ctx, serverAddr, log) + client, closer, err := d.newClient(serverAddr, log) if err != nil { return err } defer closer.Close() - log.Infof("Trying to download info") + log.Info("Trying to download info") resp, err := client.GetInfo(ctx, &pb.GetInfoRequest{}) if err != nil { return fmt.Errorf("getting info from other instance: %w", err) } - log.Infof("Successfully downloaded info") + log.Info("Successfully downloaded info") return d.info.SetProto(resp.Info) } // DownloadDeployment will open a new grpc connection to another instance, attempting to download files from that instance. func (d *Download) DownloadDeployment(ctx context.Context, ip string) error { - log := d.log.With(zap.String("ip", ip)) + log := d.log.With(slog.String("ip", ip)) serverAddr := net.JoinHostPort(ip, strconv.Itoa(constants.DebugdPort)) - client, closer, err := d.newClient(ctx, serverAddr, log) + client, closer, err := d.newClient(serverAddr, log) if err != nil { return err } defer closer.Close() - log.Infof("Trying to download files") + log.Info("Trying to download files") stream, err := client.DownloadFiles(ctx, &pb.DownloadFilesRequest{}) if err != nil { return fmt.Errorf("starting file download from other instance: %w", err) @@ -90,15 +89,15 @@ func (d *Download) DownloadDeployment(ctx context.Context, ip string) error { err = d.transfer.RecvFiles(stream) switch { case err == nil: - d.log.Infof("Downloading files succeeded") + d.log.Info("Downloading files succeeded") case errors.Is(err, filetransfer.ErrReceiveRunning): - d.log.Warnf("Download already in progress") + d.log.Warn("Download already in progress") return err case errors.Is(err, filetransfer.ErrReceiveFinished): - d.log.Warnf("Download already finished") + d.log.Warn("Download already finished") return nil default: - d.log.With(zap.Error(err)).Errorf("Downloading files failed") + d.log.With(slog.Any("error", err)).Error("Downloading files failed") return err } @@ -111,24 +110,24 @@ func (d *Download) DownloadDeployment(ctx context.Context, ip string) error { ctx, file.OverrideServiceUnit, file.TargetPath, ); err != nil { // continue on error to allow other units to be overridden - d.log.With(zap.Error(err)).Errorf("Failed to override service unit %s", file.OverrideServiceUnit) + d.log.With(slog.Any("error", err)).Error(fmt.Sprintf("Failed to override service unit %s", file.OverrideServiceUnit)) } } return nil } -func (d *Download) newClient(ctx context.Context, serverAddr string, log *logger.Logger) (pb.DebugdClient, io.Closer, error) { - log.Infof("Connecting to server") - conn, err := d.dial(ctx, serverAddr) +func (d *Download) newClient(serverAddr string, log *slog.Logger) (pb.DebugdClient, io.Closer, error) { + log.Info("Connecting to server") + conn, err := d.dial(serverAddr) if err != nil { return nil, nil, fmt.Errorf("connecting to other instance via gRPC: %w", err) } return pb.NewDebugdClient(conn), conn, nil } -func (d *Download) dial(ctx context.Context, target string) (*grpc.ClientConn, error) { - return grpc.DialContext(ctx, target, +func (d *Download) dial(target string) (*grpc.ClientConn, error) { + return grpc.NewClient(target, d.grpcWithDialer(), grpc.WithTransportCredentials(insecure.NewCredentials()), ) diff --git a/debugd/internal/debugd/deploy/download_test.go b/debugd/internal/debugd/deploy/download_test.go index c6215092d..0cd800124 100644 --- a/debugd/internal/debugd/deploy/download_test.go +++ b/debugd/internal/debugd/deploy/download_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package deploy @@ -27,6 +27,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -122,7 +123,7 @@ func TestDownloadDeployment(t *testing.T) { serviceManager: serviceMgr, } - err := download.DownloadDeployment(context.Background(), ip) + err := download.DownloadDeployment(t.Context(), ip) if tc.wantErr { assert.Error(err) @@ -193,7 +194,7 @@ func TestDownloadInfo(t *testing.T) { info: &tc.infoSetter, } - err := download.DownloadInfo(context.Background(), ip) + err := download.DownloadInfo(t.Context(), ip) if tc.wantErr { assert.Error(err) diff --git a/debugd/internal/debugd/deploy/service.go b/debugd/internal/debugd/deploy/service.go index f82e09c25..618875989 100644 --- a/debugd/internal/debugd/deploy/service.go +++ b/debugd/internal/debugd/deploy/service.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package deploy @@ -9,15 +9,14 @@ package deploy import ( "context" "fmt" + "log/slog" "os" "path/filepath" "regexp" "strings" "sync" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/afero" - "go.uber.org/zap" ) const ( @@ -60,18 +59,20 @@ type SystemdUnit struct { // ServiceManager receives ServiceManagerRequests and units via channels and performs the requests / creates the unit files. type ServiceManager struct { - log *logger.Logger + log *slog.Logger dbus dbusClient + journal journalReader fs afero.Fs systemdUnitFilewriteLock sync.Mutex } // NewServiceManager creates a new ServiceManager. -func NewServiceManager(log *logger.Logger) *ServiceManager { +func NewServiceManager(log *slog.Logger) *ServiceManager { fs := afero.NewOsFs() return &ServiceManager{ log: log, dbus: &dbusWrapper{}, + journal: &journalctlWrapper{}, fs: fs, systemdUnitFilewriteLock: sync.Mutex{}, } @@ -90,6 +91,8 @@ type dbusConn interface { // StopUnitContext is similar to StartUnitContext, but stops the specified unit // rather than starting it. StopUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) + // ResetFailedUnitContext resets the "failed" state of a unit. + ResetFailedUnitContext(ctx context.Context, name string) error // RestartUnitContext restarts a service. If a service is restarted that isn't // running it will be started. RestartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) @@ -100,9 +103,14 @@ type dbusConn interface { Close() } +type journalReader interface { + // ReadJournal reads the journal for a specific unit. + readJournal(unit string) string +} + // SystemdAction will perform a systemd action on a service unit (start, stop, restart, reload). func (s *ServiceManager) SystemdAction(ctx context.Context, request ServiceManagerRequest) error { - log := s.log.With(zap.String("unit", request.Unit), zap.String("action", request.Action.String())) + log := s.log.With(slog.String("unit", request.Unit), slog.String("action", request.Action.String())) conn, err := s.dbus.NewSystemConnectionContext(ctx) if err != nil { return fmt.Errorf("establishing systemd connection: %w", err) @@ -116,6 +124,9 @@ func (s *ServiceManager) SystemdAction(ctx context.Context, request ServiceManag case Stop: _, err = conn.StopUnitContext(ctx, request.Unit, "replace", resultChan) case Restart: + if err = conn.ResetFailedUnitContext(ctx, request.Unit); err != nil { + s.log.Error("Failed to reset unit failed state", "error", err.Error(), "unit", request.Unit) + } _, err = conn.RestartUnitContext(ctx, request.Unit, "replace", resultChan) case Reload: err = conn.ReloadContext(ctx) @@ -127,7 +138,7 @@ func (s *ServiceManager) SystemdAction(ctx context.Context, request ServiceManag } if request.Action == Reload { - log.Infof("daemon-reload succeeded") + log.Info("daemon-reload succeeded") return nil } // Wait for the action to finish and then check if it was @@ -136,18 +147,19 @@ func (s *ServiceManager) SystemdAction(ctx context.Context, request ServiceManag switch result { case "done": - log.Infof("%s on systemd unit %s succeeded", request.Action, request.Unit) + log.Info(fmt.Sprintf("%s on systemd unit %s succeeded", request.Action, request.Unit)) return nil default: - return fmt.Errorf("performing action %q on systemd unit %q failed: expected %q but received %q", request.Action.String(), request.Unit, "done", result) + serviceJournal := s.journal.readJournal(request.Unit) + return fmt.Errorf("performing action %q on systemd unit %q failed: expected %q but received %q. systemd unit journal entries: %s", request.Action.String(), request.Unit, "done", result, serviceJournal) } } // WriteSystemdUnitFile will write a systemd unit to disk. func (s *ServiceManager) WriteSystemdUnitFile(ctx context.Context, unit SystemdUnit) error { - log := s.log.With(zap.String("unitFile", fmt.Sprintf("%s/%s", systemdUnitFolder, unit.Name))) - log.Infof("Writing systemd unit file") + log := s.log.With(slog.String("unitFile", fmt.Sprintf("%s/%s", systemdUnitFolder, unit.Name))) + log.Info("Writing systemd unit file") s.systemdUnitFilewriteLock.Lock() defer s.systemdUnitFilewriteLock.Unlock() if err := afero.WriteFile(s.fs, fmt.Sprintf("%s/%s", systemdUnitFolder, unit.Name), []byte(unit.Contents), 0o644); err != nil { @@ -158,14 +170,14 @@ func (s *ServiceManager) WriteSystemdUnitFile(ctx context.Context, unit SystemdU return fmt.Errorf("performing systemd daemon-reload: %w", err) } - log.Infof("Wrote systemd unit file and performed daemon-reload") + log.Info("Wrote systemd unit file and performed daemon-reload") return nil } // OverrideServiceUnitExecStart will override the ExecStart of a systemd unit. func (s *ServiceManager) OverrideServiceUnitExecStart(ctx context.Context, unitName, execStart string) error { - log := s.log.With(zap.String("unitFile", fmt.Sprintf("%s/%s", systemdUnitFolder, unitName))) - log.Infof("Overriding systemd unit file execStart") + log := s.log.With(slog.String("unitFile", fmt.Sprintf("%s/%s", systemdUnitFolder, unitName))) + log.Info("Overriding systemd unit file execStart") if !systemdUnitNameRegexp.MatchString(unitName) { return fmt.Errorf("unit name %q is invalid", unitName) } @@ -187,13 +199,13 @@ func (s *ServiceManager) OverrideServiceUnitExecStart(ctx context.Context, unitN // do not return early here // the "daemon-reload" command may return an unrelated error // and there is no way to know if the override was successful - log.Warnf("Failed to perform systemd daemon-reload: %v", err) + log.Warn(fmt.Sprintf("Failed to perform systemd daemon-reload: %v", err)) } if err := s.SystemdAction(ctx, ServiceManagerRequest{Unit: unitName + ".service", Action: Restart}); err != nil { - log.Warnf("Failed to perform unit restart: %v", err) + log.Warn(fmt.Sprintf("Failed to perform unit restart: %v", err)) return fmt.Errorf("performing systemd unit restart: %w", err) } - log.Infof("Overrode systemd unit file execStart, performed daemon-reload and restarted unit %v", unitName) + log.Info(fmt.Sprintf("Overrode systemd unit file execStart, performed daemon-reload and restarted unit %v", unitName)) return nil } diff --git a/debugd/internal/debugd/deploy/service_test.go b/debugd/internal/debugd/deploy/service_test.go index c0c98f93e..06d9820cd 100644 --- a/debugd/internal/debugd/deploy/service_test.go +++ b/debugd/internal/debugd/deploy/service_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package deploy @@ -104,10 +104,11 @@ func TestSystemdAction(t *testing.T) { manager := ServiceManager{ log: logger.NewTest(t), dbus: &tc.dbus, + journal: &stubJournalReader{}, fs: fs, systemdUnitFilewriteLock: sync.Mutex{}, } - err := manager.SystemdAction(context.Background(), ServiceManagerRequest{ + err := manager.SystemdAction(t.Context(), ServiceManagerRequest{ Unit: unitName, Action: tc.action, }) @@ -183,10 +184,11 @@ func TestWriteSystemdUnitFile(t *testing.T) { manager := ServiceManager{ log: logger.NewTest(t), dbus: &tc.dbus, + journal: &stubJournalReader{}, fs: fs, systemdUnitFilewriteLock: sync.Mutex{}, } - err := manager.WriteSystemdUnitFile(context.Background(), tc.unit) + err := manager.WriteSystemdUnitFile(t.Context(), tc.unit) if tc.wantErr { assert.Error(err) @@ -296,10 +298,11 @@ func TestOverrideServiceUnitExecStart(t *testing.T) { manager := ServiceManager{ log: logger.NewTest(t), dbus: &tc.dbus, + journal: &stubJournalReader{}, fs: fs, systemdUnitFilewriteLock: sync.Mutex{}, } - err := manager.OverrideServiceUnitExecStart(context.Background(), tc.unitName, tc.execStart) + err := manager.OverrideServiceUnitExecStart(t.Context(), tc.unitName, tc.execStart) if tc.wantErr { assert.Error(err) @@ -353,6 +356,10 @@ func (c *fakeDbusConn) StopUnitContext(_ context.Context, name string, mode stri return c.jobID, c.actionErr } +func (c *fakeDbusConn) ResetFailedUnitContext(_ context.Context, _ string) error { + return nil +} + func (c *fakeDbusConn) RestartUnitContext(_ context.Context, name string, mode string, ch chan<- string) (int, error) { c.inputs = append(c.inputs, dbusConnActionInput{name: name, mode: mode}) ch <- c.result @@ -367,3 +374,9 @@ func (c *fakeDbusConn) ReloadContext(_ context.Context) error { } func (c *fakeDbusConn) Close() {} + +type stubJournalReader struct{} + +func (s *stubJournalReader) readJournal(_ string) string { + return "" +} diff --git a/debugd/internal/debugd/deploy/wrappers.go b/debugd/internal/debugd/deploy/wrappers.go index 9ec9f0b01..57391d6a6 100644 --- a/debugd/internal/debugd/deploy/wrappers.go +++ b/debugd/internal/debugd/deploy/wrappers.go @@ -1,13 +1,14 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package deploy import ( "context" + "os/exec" "github.com/coreos/go-systemd/v22/dbus" ) @@ -37,6 +38,10 @@ func (c *dbusConnWrapper) StopUnitContext(ctx context.Context, name string, mode return c.conn.StopUnitContext(ctx, name, mode, ch) } +func (c *dbusConnWrapper) ResetFailedUnitContext(ctx context.Context, name string) error { + return c.conn.ResetFailedUnitContext(ctx, name) +} + func (c *dbusConnWrapper) RestartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) { return c.conn.RestartUnitContext(ctx, name, mode, ch) } @@ -48,3 +53,85 @@ func (c *dbusConnWrapper) ReloadContext(ctx context.Context) error { func (c *dbusConnWrapper) Close() { c.conn.Close() } + +type journalctlWrapper struct{} + +func (j *journalctlWrapper) readJournal(unit string) string { + out, _ := exec.CommandContext(context.Background(), "journalctl", "-u", unit, "--no-pager").CombinedOutput() + return string(out) +} + +/* +// Preferably, we would use the systemd journal API directly. +// However, this requires linking against systemd libraries, so we go with the easier journalctl command for now. + +type sdJournalWrapper struct{} + +// readJournal reads the journal for a specific unit. +func (s *sdJournalWrapper) readJournal(unit string) string { + journal, err := sdjournal.NewJournal() + if err != nil { + log.Printf("opening journal: %s", err) + return "" + } + defer journal.Close() + + // Filter the journal for the specified unit + filters := []string{ + fmt.Sprintf("_SYSTEMD_UNIT=%s", unit), + fmt.Sprintf("UNIT=%s", unit), + fmt.Sprintf("OBJECT_SYSTEMD_UNIT=%s", unit), + fmt.Sprintf("_SYSTEMD_SLICE=%s", unit), + fmt.Sprintf("_SYSTEMD_USER_UNIT=%s", unit), + fmt.Sprintf("USER_UNIT=%s", unit), + fmt.Sprintf("COREDUMP_USER_UNIT=%s", unit), + fmt.Sprintf("OBJECT_SYSTEMD_USER_UNIT=%s", unit), + fmt.Sprintf("_SYSTEMD_USER_SLICE=%s", unit), + } + for _, filter := range filters { + if err := journal.AddMatch(filter); err != nil { + log.Printf("applying filter %q: %s", filter, err) + return "" + } + if err := journal.AddDisjunction(); err != nil { + log.Printf("adding disjunction to journal filter: %s", err) + return "" + } + } + + // Seek to the beginning of the journal + if err := journal.SeekHead(); err != nil { + log.Printf("seeking journal tail: %s", err) + return "" + } + + // Iterate over the journal entries + var previousCursor string + journalLog := &strings.Builder{} + for { + if _, err := journal.Next(); err != nil { + log.Printf("getting next entry in journal: %s", err) + return "" + } + + entry, err := journal.GetEntry() + if err != nil { + log.Printf("getting journal entry: %s", err) + return "" + } + + // Abort if we reached the end of the journal, i.e. the cursor didn't change + if entry.Cursor == previousCursor { + break + } + previousCursor = entry.Cursor + + if _, err := journalLog.WriteString(entry.Fields[sdjournal.SD_JOURNAL_FIELD_MESSAGE] + "\n"); err != nil { + log.Printf("copying journal entry to buffer: %s", err) + return "" + } + } + + return strings.TrimSpace(journalLog.String()) +} +*/ diff --git a/debugd/internal/debugd/info/info.go b/debugd/internal/debugd/info/info.go index 06df4c71c..a9a193734 100644 --- a/debugd/internal/debugd/info/info.go +++ b/debugd/internal/debugd/info/info.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package info implements the info map that is diff --git a/debugd/internal/debugd/info/info_test.go b/debugd/internal/debugd/info/info_test.go index 9c129ff6a..a2b38422d 100644 --- a/debugd/internal/debugd/info/info_test.go +++ b/debugd/internal/debugd/info/info_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package info diff --git a/debugd/internal/debugd/logcollector/BUILD.bazel b/debugd/internal/debugd/logcollector/BUILD.bazel index 6ecc05a71..9fcc765b3 100644 --- a/debugd/internal/debugd/logcollector/BUILD.bazel +++ b/debugd/internal/debugd/logcollector/BUILD.bazel @@ -14,12 +14,11 @@ go_library( "//debugd/internal/debugd/info", "//internal/cloud/cloudprovider", "//internal/cloud/metadata", - "//internal/logger", "//internal/versions", "@com_github_aws_aws_sdk_go_v2_config//:config", "@com_github_aws_aws_sdk_go_v2_service_secretsmanager//:secretsmanager", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", - "@com_github_azure_azure_sdk_for_go_sdk_keyvault_azsecrets//:azsecrets", + "@com_github_azure_azure_sdk_for_go_sdk_security_keyvault_azsecrets//:azsecrets", "@com_github_googleapis_gax_go_v2//:gax-go", "@com_google_cloud_go_secretmanager//apiv1", "@com_google_cloud_go_secretmanager//apiv1/secretmanagerpb", @@ -32,7 +31,7 @@ go_test( embed = [":logcollector"], deps = [ "@com_github_aws_aws_sdk_go_v2_service_secretsmanager//:secretsmanager", - "@com_github_azure_azure_sdk_for_go_sdk_keyvault_azsecrets//:azsecrets", + "@com_github_azure_azure_sdk_for_go_sdk_security_keyvault_azsecrets//:azsecrets", "@com_github_googleapis_gax_go_v2//:gax-go", "@com_github_stretchr_testify//assert", "@com_google_cloud_go_secretmanager//apiv1/secretmanagerpb", diff --git a/debugd/internal/debugd/logcollector/credentials.go b/debugd/internal/debugd/logcollector/credentials.go index e987af1ea..03b94b174 100644 --- a/debugd/internal/debugd/logcollector/credentials.go +++ b/debugd/internal/debugd/logcollector/credentials.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package logcollector @@ -17,7 +17,7 @@ import ( gcpsecretmanager "cloud.google.com/go/secretmanager/apiv1" gcpsecretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" awsconfig "github.com/aws/aws-sdk-go-v2/config" awssecretmanager "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/info" @@ -86,7 +86,7 @@ func newGCPCloudCredentialGetter(ctx context.Context) (*gcpCloudCredentialGetter } func (g *gcpCloudCredentialGetter) GetOpensearchCredentials(ctx context.Context) (credentials, error) { - const secretName = "projects/796962942582/secrets/e2e-logs-OpenSearch-password/versions/1" + const secretName = "projects/1052692473304/secrets/e2e-logs-OpenSearch-password/versions/1" const username = "cluster-instance-gcp" req := &gcpsecretmanagerpb.AccessSecretVersionRequest{Name: secretName} diff --git a/debugd/internal/debugd/logcollector/credentials_test.go b/debugd/internal/debugd/logcollector/credentials_test.go index a48f5a800..4bef5d86f 100644 --- a/debugd/internal/debugd/logcollector/credentials_test.go +++ b/debugd/internal/debugd/logcollector/credentials_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package logcollector @@ -13,7 +13,7 @@ import ( "testing" "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" - "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" awssecretmanager "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/googleapis/gax-go/v2" "github.com/stretchr/testify/assert" @@ -67,7 +67,7 @@ func TestGetOpensearchCredentialsGCP(t *testing.T) { g := &gcpCloudCredentialGetter{secretsAPI: tc.gcpAPI} - gotCreds, err := g.GetOpensearchCredentials(context.Background()) + gotCreds, err := g.GetOpensearchCredentials(t.Context()) if tc.wantErr { assert.Error(err) @@ -103,7 +103,7 @@ func TestGetOpensearchCredentialsAzure(t *testing.T) { "azure success": { azureAPI: stubAzureSecretsAPI{ getSecretResp: azsecrets.GetSecretResponse{ - SecretBundle: azsecrets.SecretBundle{ + Secret: azsecrets.Secret{ Value: ptr("test-password"), }, }, @@ -127,7 +127,7 @@ func TestGetOpensearchCredentialsAzure(t *testing.T) { a := &azureCloudCredentialGetter{secretsAPI: tc.azureAPI} - gotCreds, err := a.GetOpensearchCredentials(context.Background()) + gotCreds, err := a.GetOpensearchCredentials(t.Context()) if tc.wantErr { assert.Error(err) @@ -184,7 +184,7 @@ func TestGetOpensearchCredentialsAWS(t *testing.T) { a := &awsCloudCredentialGetter{secretmanager: tc.awsAPI} - gotCreds, err := a.GetOpensearchCredentials(context.Background()) + gotCreds, err := a.GetOpensearchCredentials(t.Context()) if tc.wantErr { assert.Error(err) diff --git a/debugd/internal/debugd/logcollector/fields.go b/debugd/internal/debugd/logcollector/fields.go index 08c3776d7..e35864c72 100644 --- a/debugd/internal/debugd/logcollector/fields.go +++ b/debugd/internal/debugd/logcollector/fields.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package logcollector @@ -32,11 +32,12 @@ var ( // cloud provider used in e2e test. If deployed with debugd, this is a duplicate as its also // available in the metadata. If deployed through K8s in e2e tests with a stable image, this // is where the cloud provider is saved in. - "github.e2e-test-provider": {}, - "github.ref-stream": {}, - "github.kubernetes-version": {}, - "github.cluster-creation": {}, - "deployment-type": {}, // deployment type, e.g. "debugd", "k8s" + "github.e2e-test-provider": {}, + "github.ref-stream": {}, + "github.kubernetes-version": {}, + "github.cluster-creation": {}, + "github.attestation-variant": {}, + "deployment-type": {}, // deployment type, e.g. "debugd", "k8s" } ) diff --git a/debugd/internal/debugd/logcollector/logcollector.go b/debugd/internal/debugd/logcollector/logcollector.go index 9e5c60165..809133ad0 100644 --- a/debugd/internal/debugd/logcollector/logcollector.go +++ b/debugd/internal/debugd/logcollector/logcollector.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package logcollector uses podman to deploy logstash and filebeat containers @@ -11,7 +11,7 @@ package logcollector import ( "context" "fmt" - "io" + "log/slog" "os" "os/exec" "path/filepath" @@ -22,7 +22,6 @@ import ( "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/info" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/versions" ) @@ -36,60 +35,60 @@ const ( // // This requires podman to be installed. func NewStartTrigger(ctx context.Context, wg *sync.WaitGroup, provider cloudprovider.Provider, - metadata providerMetadata, logger *logger.Logger, + metadata providerMetadata, logger *slog.Logger, ) func(*info.Map) { return func(infoMap *info.Map) { wg.Add(1) go func() { defer wg.Done() - logger.Infof("Start trigger running") + logger.Info("Start trigger running") if err := ctx.Err(); err != nil { - logger.With("err", err).Errorf("Start trigger canceled") + logger.With("err", err).Error("Start trigger canceled") return } - logger.Infof("Get flags from infos") + logger.Info("Get flags from infos") _, ok, err := infoMap.Get("logcollect") if err != nil { - logger.Errorf("Getting infos: %v", err) + logger.Error(fmt.Sprintf("Getting infos: %v", err)) return } if !ok { - logger.Infof("Flag 'logcollect' not set") + logger.Info("Flag 'logcollect' not set") return } cerdsGetter, err := newCloudCredentialGetter(ctx, provider, infoMap) if err != nil { - logger.Errorf("Creating cloud credential getter: %v", err) + logger.Error(fmt.Sprintf("Creating cloud credential getter: %v", err)) return } - logger.Infof("Getting credentials") + logger.Info("Getting credentials") creds, err := cerdsGetter.GetOpensearchCredentials(ctx) if err != nil { - logger.Errorf("Getting opensearch credentials: %v", err) + logger.Error(fmt.Sprintf("Getting opensearch credentials: %v", err)) return } - logger.Infof("Getting logstash pipeline template from image %s", versions.LogstashImage) + logger.Info(fmt.Sprintf("Getting logstash pipeline template from image %s", versions.LogstashImage)) tmpl, err := getTemplate(ctx, logger, versions.LogstashImage, "/run/logstash/templates/pipeline.conf", "/run/logstash") if err != nil { - logger.Errorf("Getting logstash pipeline template: %v", err) + logger.Error(fmt.Sprintf("Getting logstash pipeline template: %v", err)) return } infoMapM, err := infoMap.GetCopy() if err != nil { - logger.Errorf("Getting copy of map from info: %v", err) + logger.Error(fmt.Sprintf("Getting copy of map from info: %v", err)) return } infoMapM = filterInfoMap(infoMapM) setCloudMetadata(ctx, infoMapM, provider, metadata) - logger.Infof("Writing logstash pipeline") + logger.Info("Writing logstash pipeline") pipelineConf := logstashConfInput{ Port: 5044, Host: openSearchHost, @@ -97,14 +96,14 @@ func NewStartTrigger(ctx context.Context, wg *sync.WaitGroup, provider cloudprov Credentials: creds, } if err := writeTemplate("/run/logstash/pipeline/pipeline.conf", tmpl, pipelineConf); err != nil { - logger.Errorf("Writing logstash config: %v", err) + logger.Error(fmt.Sprintf("Writing logstash config: %v", err)) return } - logger.Infof("Getting filebeat config template from image %s", versions.FilebeatImage) + logger.Info(fmt.Sprintf("Getting filebeat config template from image %s", versions.FilebeatImage)) tmpl, err = getTemplate(ctx, logger, versions.FilebeatImage, "/run/filebeat/templates/filebeat.yml", "/run/filebeat") if err != nil { - logger.Errorf("Getting filebeat config template: %v", err) + logger.Error(fmt.Sprintf("Getting filebeat config template: %v", err)) return } filebeatConf := filebeatConfInput{ @@ -112,43 +111,26 @@ func NewStartTrigger(ctx context.Context, wg *sync.WaitGroup, provider cloudprov AddCloudMetadata: true, } if err := writeTemplate("/run/filebeat/filebeat.yml", tmpl, filebeatConf); err != nil { - logger.Errorf("Writing filebeat pipeline: %v", err) + logger.Error(fmt.Sprintf("Writing filebeat pipeline: %v", err)) return } - logger.Infof("Getting metricbeat config template from image %s", versions.MetricbeatImage) - tmpl, err = getTemplate(ctx, logger, versions.MetricbeatImage, "/run/metricbeat/templates/metricbeat.yml", "/run/metricbeat") - if err != nil { - logger.Errorf("Getting metricbeat config template: %v", err) - return - } - metricbeatConf := metricbeatConfInput{ - LogstashHost: "localhost:5044", - Port: 5066, - CollectSystemMetrics: true, - AddCloudMetadata: true, - } - if err := writeTemplate("/run/metricbeat/metricbeat.yml", tmpl, metricbeatConf); err != nil { - logger.Errorf("Writing metricbeat pipeline: %v", err) - return - } - - logger.Infof("Starting log collection pod") + logger.Info("Starting log collection pod") if err := startPod(ctx, logger); err != nil { - logger.Errorf("Starting log collection: %v", err) + logger.Error(fmt.Sprintf("Starting log collection: %v", err)) } }() } } -func getTemplate(ctx context.Context, logger *logger.Logger, image, templateDir, destDir string) (*template.Template, error) { +func getTemplate(ctx context.Context, logger *slog.Logger, image, templateDir, destDir string) (*template.Template, error) { createContainerArgs := []string{ "create", "--name=template", image, } - createContainerCmd := exec.CommandContext(ctx, "podman", createContainerArgs...) - logger.Infof("Creating template container") + createContainerCmd := podman(ctx, createContainerArgs...) + logger.Info("Creating template container") if out, err := createContainerCmd.CombinedOutput(); err != nil { return nil, fmt.Errorf("creating template container: %w\n%s", err, out) } @@ -162,8 +144,8 @@ func getTemplate(ctx context.Context, logger *logger.Logger, image, templateDir, "template:/usr/share/constellogs/templates/", destDir, } - copyFromCmd := exec.CommandContext(ctx, "podman", copyFromArgs...) - logger.Infof("Copying templates") + copyFromCmd := podman(ctx, copyFromArgs...) + logger.Info("Copying templates") if out, err := copyFromCmd.CombinedOutput(); err != nil { return nil, fmt.Errorf("copying templates: %w\n%s", err, out) } @@ -172,8 +154,8 @@ func getTemplate(ctx context.Context, logger *logger.Logger, image, templateDir, "rm", "template", } - removeContainerCmd := exec.CommandContext(ctx, "podman", removeContainerArgs...) - logger.Infof("Removing template container") + removeContainerCmd := podman(ctx, removeContainerArgs...) + logger.Info("Removing template container") if out, err := removeContainerCmd.CombinedOutput(); err != nil { return nil, fmt.Errorf("removing template container: %w\n%s", err, out) } @@ -186,47 +168,50 @@ func getTemplate(ctx context.Context, logger *logger.Logger, image, templateDir, return tmpl, nil } -func startPod(ctx context.Context, logger *logger.Logger) error { +func startPod(ctx context.Context, logger *slog.Logger) error { // create a shared pod for filebeat, metricbeat and logstash createPodArgs := []string{ "pod", "create", "logcollection", } - createPodCmd := exec.CommandContext(ctx, "podman", createPodArgs...) - logger.Infof("Create pod command: %v", createPodCmd.String()) + createPodCmd := podman(ctx, createPodArgs...) + logger.Info(fmt.Sprintf("Create pod command: %v", createPodCmd.String())) if out, err := createPodCmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create pod: %w; output: %s", err, out) } // start logstash container - logstashLog := newCmdLogger(logger.Named("logstash")) runLogstashArgs := []string{ "run", - "--rm", + "-d", + "--restart=unless-stopped", "--name=logstash", "--pod=logcollection", - "--log-driver=none", + "--log-driver=journald", "--volume=/run/logstash/pipeline:/usr/share/logstash/pipeline/:ro", versions.LogstashImage, } - runLogstashCmd := exec.CommandContext(ctx, "podman", runLogstashArgs...) - logger.Infof("Run logstash command: %v", runLogstashCmd.String()) - runLogstashCmd.Stdout = logstashLog - runLogstashCmd.Stderr = logstashLog - if err := runLogstashCmd.Start(); err != nil { + runLogstashCmd := podman(ctx, runLogstashArgs...) + logger.Info(fmt.Sprintf("Run logstash command: %v", runLogstashCmd.String())) + if out, err := runLogstashCmd.CombinedOutput(); err != nil { + logger.Error("Could not start logstash container", "err", err, "output", out) return fmt.Errorf("failed to start logstash: %w", err) } + if out, err := podman(ctx, "wait", "logstash", "--condition=running", "--interval=15s").CombinedOutput(); err != nil { + logger.Error("Logstash container failed to reach healthy status", "err", err, "output", out) + return fmt.Errorf("waiting for logstash container to reach healthy status: %w; output: %s", err, out) + } // start filebeat container - filebeatLog := newCmdLogger(logger.Named("filebeat")) runFilebeatArgs := []string{ "run", - "--rm", + "-d", + "--restart=unless-stopped", "--name=filebeat", "--pod=logcollection", "--privileged", - "--log-driver=none", + "--log-driver=journald", "--volume=/run/log/journal:/run/log/journal:ro", "--volume=/etc/machine-id:/etc/machine-id:ro", "--volume=/run/systemd:/run/systemd:ro", @@ -235,34 +220,15 @@ func startPod(ctx context.Context, logger *logger.Logger) error { "--volume=/run/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro", versions.FilebeatImage, } - runFilebeatCmd := exec.CommandContext(ctx, "podman", runFilebeatArgs...) - logger.Infof("Run filebeat command: %v", runFilebeatCmd.String()) - runFilebeatCmd.Stdout = filebeatLog - runFilebeatCmd.Stderr = filebeatLog - if err := runFilebeatCmd.Start(); err != nil { + runFilebeatCmd := podman(ctx, runFilebeatArgs...) + logger.Info(fmt.Sprintf("Run filebeat command: %v", runFilebeatCmd.String())) + if out, err := runFilebeatCmd.CombinedOutput(); err != nil { + logger.Error("Could not start filebeat container", "err", err, "output", out) return fmt.Errorf("failed to run filebeat: %w", err) } - - // start metricbeat container - metricbeatLog := newCmdLogger(logger.Named("metricbeat")) - runMetricbeatArgs := []string{ - "run", - "--rm", - "--name=metricbeat", - "--pod=logcollection", - "--privileged", - "--log-driver=none", - "--volume=/proc:/hostfs/proc:ro", - "--volume=/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro", - "--volume=/run/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro", - versions.MetricbeatImage, - } - runMetricbeatCmd := exec.CommandContext(ctx, "podman", runMetricbeatArgs...) - logger.Infof("Run metricbeat command: %v", runMetricbeatCmd.String()) - runMetricbeatCmd.Stdout = metricbeatLog - runMetricbeatCmd.Stderr = metricbeatLog - if err := runMetricbeatCmd.Start(); err != nil { - return fmt.Errorf("failed to run metricbeat: %w", err) + if out, err := podman(ctx, "wait", "filebeat", "--condition=running", "--interval=15s").CombinedOutput(); err != nil { + logger.Error("Filebeat container failed to reach healthy status", "err", err, "output", out) + return fmt.Errorf("waiting for filebeat container to reach healthy status: %w; output: %s", err, out) } return nil @@ -280,14 +246,6 @@ type filebeatConfInput struct { AddCloudMetadata bool } -type metricbeatConfInput struct { - Port int - LogstashHost string - CollectEtcdMetrics bool - CollectSystemMetrics bool - AddCloudMetadata bool -} - func writeTemplate(path string, templ *template.Template, in any) error { if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil { return fmt.Errorf("creating template dir: %w", err) @@ -342,17 +300,9 @@ func setCloudMetadata(ctx context.Context, m map[string]string, provider cloudpr } } -func newCmdLogger(logger *logger.Logger) io.Writer { - return &cmdLogger{logger: logger} -} - -type cmdLogger struct { - logger *logger.Logger -} - -func (c *cmdLogger) Write(p []byte) (n int, err error) { - c.logger.Infof("%s", p) - return len(p), nil +func podman(ctx context.Context, args ...string) *exec.Cmd { + args = append([]string{"--runtime=runc"}, args...) + return exec.CommandContext(ctx, "podman", args...) } type providerMetadata interface { diff --git a/debugd/internal/debugd/metadata/BUILD.bazel b/debugd/internal/debugd/metadata/BUILD.bazel index 8cc159b11..7b57a5fb0 100644 --- a/debugd/internal/debugd/metadata/BUILD.bazel +++ b/debugd/internal/debugd/metadata/BUILD.bazel @@ -9,11 +9,7 @@ go_library( ], importpath = "github.com/edgelesssys/constellation/v2/debugd/internal/debugd/metadata", visibility = ["//debugd:__subpackages__"], - deps = [ - "//debugd/internal/debugd", - "//internal/logger", - "@org_uber_go_zap//:zap", - ], + deps = ["//debugd/internal/debugd"], ) go_test( diff --git a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go index 64a19aa3d..52a7d09d3 100644 --- a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go +++ b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package cloudprovider implements a metadata service for cloud providers. diff --git a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go index 53eb3ad8e..52c29e1b3 100644 --- a/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go +++ b/debugd/internal/debugd/metadata/cloudprovider/cloudprovider_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudprovider @@ -22,6 +22,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -55,7 +56,7 @@ func TestRole(t *testing.T) { fetcher := Fetcher{tc.meta} - role, err := fetcher.Role(context.Background()) + role, err := fetcher.Role(t.Context()) if tc.wantErr { assert.Error(err) @@ -109,7 +110,7 @@ func TestDiscoverDebugIPs(t *testing.T) { fetcher := Fetcher{ metaAPI: &tc.meta, } - ips, err := fetcher.DiscoverDebugdIPs(context.Background()) + ips, err := fetcher.DiscoverDebugdIPs(t.Context()) if tc.wantErr { assert.Error(err) @@ -148,7 +149,7 @@ func TestDiscoverLoadBalancerIP(t *testing.T) { metaAPI: tc.metaAPI, } - ip, err := fetcher.DiscoverLoadBalancerIP(context.Background()) + ip, err := fetcher.DiscoverLoadBalancerIP(t.Context()) if tc.wantErr { assert.Error(err) diff --git a/debugd/internal/debugd/metadata/fallback/fallback.go b/debugd/internal/debugd/metadata/fallback/fallback.go index 39308390f..9b60a1a77 100644 --- a/debugd/internal/debugd/metadata/fallback/fallback.go +++ b/debugd/internal/debugd/metadata/fallback/fallback.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package fallback implements a fake metadata backend. diff --git a/debugd/internal/debugd/metadata/fallback/fallback_test.go b/debugd/internal/debugd/metadata/fallback/fallback_test.go index cfb35582b..c00fb5893 100644 --- a/debugd/internal/debugd/metadata/fallback/fallback_test.go +++ b/debugd/internal/debugd/metadata/fallback/fallback_test.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package fallback import ( - "context" "testing" "github.com/edgelesssys/constellation/v2/internal/role" @@ -16,26 +15,26 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestDiscoverDebugdIPs(t *testing.T) { assert := assert.New(t) fetcher := NewFallbackFetcher() - ips, err := fetcher.DiscoverDebugdIPs(context.Background()) + ips, err := fetcher.DiscoverDebugdIPs(t.Context()) assert.NoError(err) assert.Empty(ips) - rol, err := fetcher.Role(context.Background()) + rol, err := fetcher.Role(t.Context()) assert.NoError(err) assert.Equal(rol, role.Unknown) - uid, err := fetcher.UID(context.Background()) + uid, err := fetcher.UID(t.Context()) assert.NoError(err) assert.Empty(uid) - self, err := fetcher.Self(context.Background()) + self, err := fetcher.Self(t.Context()) assert.NoError(err) assert.Empty(self) } diff --git a/debugd/internal/debugd/metadata/metadata.go b/debugd/internal/debugd/metadata/metadata.go index cecbff67a..814e3d7f4 100644 --- a/debugd/internal/debugd/metadata/metadata.go +++ b/debugd/internal/debugd/metadata/metadata.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package metadata schedules the discovery of other debugd instances diff --git a/debugd/internal/debugd/metadata/scheduler.go b/debugd/internal/debugd/metadata/scheduler.go index eb04e5ade..e7352fb73 100644 --- a/debugd/internal/debugd/metadata/scheduler.go +++ b/debugd/internal/debugd/metadata/scheduler.go @@ -1,19 +1,18 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package metadata import ( "context" + "log/slog" "sync" "time" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd" - "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" ) // Fetcher retrieves other debugd IPs from cloud provider metadata. @@ -24,7 +23,7 @@ type Fetcher interface { // Scheduler schedules fetching of metadata using timers. type Scheduler struct { - log *logger.Logger + log *slog.Logger fetcher Fetcher downloader downloader deploymentDone bool @@ -33,7 +32,7 @@ type Scheduler struct { } // NewScheduler returns a new scheduler. -func NewScheduler(log *logger.Logger, fetcher Fetcher, downloader downloader) *Scheduler { +func NewScheduler(log *slog.Logger, fetcher Fetcher, downloader downloader) *Scheduler { return &Scheduler{ log: log, fetcher: fetcher, @@ -60,22 +59,22 @@ func (s *Scheduler) Start(ctx context.Context, wg *sync.WaitGroup) { ips, err := s.fetcher.DiscoverDebugdIPs(ctx) if err != nil { - s.log.With(zap.Error(err)).Warnf("Discovering debugd IPs failed") + s.log.With(slog.Any("error", err)).Warn("Discovering debugd IPs failed") } lbip, err := s.fetcher.DiscoverLoadBalancerIP(ctx) if err != nil { - s.log.With(zap.Error(err)).Warnf("Discovering load balancer IP failed") + s.log.With(slog.Any("error", err)).Warn("Discovering load balancer IP failed") } else { ips = append(ips, lbip) } if len(ips) == 0 { - s.log.With(zap.Error(err)).Warnf("No debugd IPs discovered") + s.log.With(slog.Any("error", err)).Warn("No debugd IPs discovered") continue } - s.log.With(zap.Strings("ips", ips)).Infof("Discovered instances") + s.log.With(slog.Any("ips", ips)).Info("Discovered instances") s.download(ctx, ips) if s.deploymentDone && s.infoDone { return @@ -90,8 +89,8 @@ func (s *Scheduler) download(ctx context.Context, ips []string) { for _, ip := range ips { if !s.deploymentDone { if err := s.downloader.DownloadDeployment(ctx, ip); err != nil { - s.log.With(zap.Error(err), zap.String("peer", ip)). - Warnf("Downloading deployment from %s: %s", ip, err) + s.log.With(slog.Any("error", err), slog.String("peer", ip)). + Warn("Downloading deployment from %s: %s", ip, err) } else { s.deploymentDone = true } @@ -99,8 +98,8 @@ func (s *Scheduler) download(ctx context.Context, ips []string) { if !s.infoDone { if err := s.downloader.DownloadInfo(ctx, ip); err != nil { - s.log.With(zap.Error(err), zap.String("peer", ip)). - Warnf("Downloading info from %s: %s", ip, err) + s.log.With(slog.Any("error", err), slog.String("peer", ip)). + Warn("Downloading info from %s: %s", ip, err) } else { s.infoDone = true } diff --git a/debugd/internal/debugd/metadata/scheduler_test.go b/debugd/internal/debugd/metadata/scheduler_test.go index 810207927..165022837 100644 --- a/debugd/internal/debugd/metadata/scheduler_test.go +++ b/debugd/internal/debugd/metadata/scheduler_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package metadata @@ -19,7 +19,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestSchedulerStart(t *testing.T) { @@ -91,7 +91,7 @@ func TestSchedulerStart(t *testing.T) { } wg := &sync.WaitGroup{} - scheduler.Start(context.Background(), wg) + scheduler.Start(t.Context(), wg) wg.Wait() assert.Equal(tc.wantDeploymentDownloads, tc.downloader.downloadDeploymentIPs) diff --git a/debugd/internal/debugd/server/BUILD.bazel b/debugd/internal/debugd/server/BUILD.bazel index 19e646cf7..6d8d298be 100644 --- a/debugd/internal/debugd/server/BUILD.bazel +++ b/debugd/internal/debugd/server/BUILD.bazel @@ -13,9 +13,8 @@ go_library( "//debugd/service", "//internal/constants", "//internal/logger", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//keepalive", - "@org_uber_go_zap//:zap", ], ) @@ -33,7 +32,7 @@ go_test( "//internal/logger", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//credentials/insecure", "@org_uber_go_goleak//:goleak", ], diff --git a/debugd/internal/debugd/server/server.go b/debugd/internal/debugd/server/server.go index 551230ae7..ef0763a10 100644 --- a/debugd/internal/debugd/server/server.go +++ b/debugd/internal/debugd/server/server.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package server implements the gRPC endpoint of Constellation's debugd. @@ -10,7 +10,9 @@ package server import ( "context" "errors" + "log/slog" "net" + "os" "strconv" "sync" "time" @@ -21,13 +23,12 @@ import ( pb "github.com/edgelesssys/constellation/v2/debugd/service" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" ) type debugdServer struct { - log *logger.Logger + log *slog.Logger serviceManager serviceManager transfer fileTransferer info *info.Map @@ -36,7 +37,7 @@ type debugdServer struct { } // New creates a new debugdServer according to the gRPC spec. -func New(log *logger.Logger, serviceManager serviceManager, transfer fileTransferer, infos *info.Map) pb.DebugdServer { +func New(log *slog.Logger, serviceManager serviceManager, transfer fileTransferer, infos *info.Map) pb.DebugdServer { return &debugdServer{ log: log, serviceManager: serviceManager, @@ -47,25 +48,25 @@ func New(log *logger.Logger, serviceManager serviceManager, transfer fileTransfe // SetInfo sets the info of the debugd instance. func (s *debugdServer) SetInfo(_ context.Context, req *pb.SetInfoRequest) (*pb.SetInfoResponse, error) { - s.log.Infof("Received SetInfo request") + s.log.Info("Received SetInfo request") if len(req.Info) == 0 { - s.log.Infof("Info is empty") + s.log.Info("Info is empty") } setProtoErr := s.info.SetProto(req.Info) if errors.Is(setProtoErr, info.ErrInfoAlreadySet) { - s.log.Warnf("Setting info failed (already set)") + s.log.Warn("Setting info failed (already set)") return &pb.SetInfoResponse{ Status: pb.SetInfoStatus_SET_INFO_ALREADY_SET, }, nil } if setProtoErr != nil { - s.log.With(zap.Error(setProtoErr)).Errorf("Setting info failed") + s.log.With(slog.Any("error", setProtoErr)).Error("Setting info failed") return nil, setProtoErr } - s.log.Infof("Info set") + s.log.Info("Info set") return &pb.SetInfoResponse{ Status: pb.SetInfoStatus_SET_INFO_SUCCESS, @@ -74,7 +75,7 @@ func (s *debugdServer) SetInfo(_ context.Context, req *pb.SetInfoRequest) (*pb.S // GetInfo returns the info of the debugd instance. func (s *debugdServer) GetInfo(_ context.Context, _ *pb.GetInfoRequest) (*pb.GetInfoResponse, error) { - s.log.Infof("Received GetInfo request") + s.log.Info("Received GetInfo request") info, err := s.info.GetProto() if err != nil { @@ -86,25 +87,26 @@ func (s *debugdServer) GetInfo(_ context.Context, _ *pb.GetInfoRequest) (*pb.Get // UploadFiles receives a stream of files (each consisting of a header and a stream of chunks) and writes them to the filesystem. func (s *debugdServer) UploadFiles(stream pb.Debugd_UploadFilesServer) error { - s.log.Infof("Received UploadFiles request") + s.log.Info("Received UploadFiles request") err := s.transfer.RecvFiles(stream) switch { case err == nil: - s.log.Infof("Uploading files succeeded") + s.log.Info("Uploading files succeeded") case errors.Is(err, filetransfer.ErrReceiveRunning): - s.log.Warnf("Upload already in progress") + s.log.Warn("Upload already in progress") return stream.SendAndClose(&pb.UploadFilesResponse{ Status: pb.UploadFilesStatus_UPLOAD_FILES_ALREADY_STARTED, }) case errors.Is(err, filetransfer.ErrReceiveFinished): - s.log.Warnf("Upload already finished") + s.log.Warn("Upload already finished") return stream.SendAndClose(&pb.UploadFilesResponse{ Status: pb.UploadFilesStatus_UPLOAD_FILES_ALREADY_FINISHED, }) default: - s.log.With(zap.Error(err)).Errorf("Uploading files failed") + s.log.With(slog.Any("error", err)).Error("Uploading files failed") return stream.SendAndClose(&pb.UploadFilesResponse{ Status: pb.UploadFilesStatus_UPLOAD_FILES_UPLOAD_FAILED, + Error: err.Error(), }) } @@ -120,9 +122,10 @@ func (s *debugdServer) UploadFiles(stream pb.Debugd_UploadFilesServer) error { } if overrideUnitErr != nil { - s.log.With(zap.Error(overrideUnitErr)).Errorf("Overriding service units failed") + s.log.With(slog.Any("error", overrideUnitErr)).Error("Overriding service units failed") return stream.SendAndClose(&pb.UploadFilesResponse{ Status: pb.UploadFilesStatus_UPLOAD_FILES_START_FAILED, + Error: overrideUnitErr.Error(), }) } return stream.SendAndClose(&pb.UploadFilesResponse{ @@ -132,13 +135,13 @@ func (s *debugdServer) UploadFiles(stream pb.Debugd_UploadFilesServer) error { // DownloadFiles streams the previously received files to other instances. func (s *debugdServer) DownloadFiles(_ *pb.DownloadFilesRequest, stream pb.Debugd_DownloadFilesServer) error { - s.log.Infof("Sending files to other instance") + s.log.Info("Sending files to other instance") return s.transfer.SendFiles(stream) } // UploadSystemServiceUnits receives systemd service units, writes them to a service file and schedules a daemon-reload. func (s *debugdServer) UploadSystemServiceUnits(ctx context.Context, in *pb.UploadSystemdServiceUnitsRequest) (*pb.UploadSystemdServiceUnitsResponse, error) { - s.log.Infof("Uploading systemd service units") + s.log.Info("Uploading systemd service units") for _, unit := range in.Units { if err := s.serviceManager.WriteSystemdUnitFile(ctx, deploy.SystemdUnit{Name: unit.Name, Contents: unit.Contents}); err != nil { return &pb.UploadSystemdServiceUnitsResponse{Status: pb.UploadSystemdServiceUnitsStatus_UPLOAD_SYSTEMD_SERVICE_UNITS_FAILURE}, nil @@ -149,25 +152,26 @@ func (s *debugdServer) UploadSystemServiceUnits(ctx context.Context, in *pb.Uplo } // Start will start the gRPC server as goroutine. -func Start(log *logger.Logger, wg *sync.WaitGroup, serv pb.DebugdServer) { +func Start(log *slog.Logger, wg *sync.WaitGroup, serv pb.DebugdServer) { wg.Add(1) go func() { defer wg.Done() - grpcLog := log.Named("gRPC") - grpcLog.WithIncreasedLevel(zap.WarnLevel).ReplaceGRPCLogger() + grpcLog := logger.GRPCLogger(log) + logger.ReplaceGRPCLogger(grpcLog) grpcServer := grpc.NewServer( - grpcLog.GetServerStreamInterceptor(), - grpcLog.GetServerUnaryInterceptor(), + logger.GetServerStreamInterceptor(grpcLog), + logger.GetServerUnaryInterceptor(grpcLog), grpc.KeepaliveParams(keepalive.ServerParameters{Time: 15 * time.Second}), ) pb.RegisterDebugdServer(grpcServer, serv) lis, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(constants.DebugdPort))) if err != nil { - log.With(zap.Error(err)).Fatalf("Listening failed") + log.With(slog.Any("error", err)).Error("Listening failed") + os.Exit(1) } - log.Infof("gRPC server is waiting for connections") + log.Info("gRPC server is waiting for connections") grpcServer.Serve(lis) }() } diff --git a/debugd/internal/debugd/server/server_test.go b/debugd/internal/debugd/server/server_test.go index 5837a5071..a340c425f 100644 --- a/debugd/internal/debugd/server/server_test.go +++ b/debugd/internal/debugd/server/server_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package server @@ -29,7 +29,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestSetInfo(t *testing.T) { @@ -79,7 +79,7 @@ func TestSetInfo(t *testing.T) { defer conn.Close() client := pb.NewDebugdClient(conn) - setInfoStatus, err := client.SetInfo(context.Background(), &pb.SetInfoRequest{Info: tc.setInfo}) + setInfoStatus, err := client.SetInfo(t.Context(), &pb.SetInfoRequest{Info: tc.setInfo}) grpcServ.GracefulStop() assert.NoError(err) @@ -137,7 +137,7 @@ func TestGetInfo(t *testing.T) { defer conn.Close() client := pb.NewDebugdClient(conn) - resp, err := client.GetInfo(context.Background(), &pb.GetInfoRequest{}) + resp, err := client.GetInfo(t.Context(), &pb.GetInfoRequest{}) grpcServ.GracefulStop() if tc.wantErr { @@ -201,7 +201,7 @@ func TestUploadFiles(t *testing.T) { require.NoError(err) defer conn.Close() client := pb.NewDebugdClient(conn) - stream, err := client.UploadFiles(context.Background()) + stream, err := client.UploadFiles(t.Context()) require.NoError(err) resp, err := stream.CloseAndRecv() @@ -245,7 +245,7 @@ func TestDownloadFiles(t *testing.T) { require.NoError(err) defer conn.Close() client := pb.NewDebugdClient(conn) - stream, err := client.DownloadFiles(context.Background(), tc.request) + stream, err := client.DownloadFiles(t.Context(), tc.request) require.NoError(err) _, recvErr := stream.Recv() if tc.wantRecvErr { @@ -324,7 +324,7 @@ func TestUploadSystemServiceUnits(t *testing.T) { require.NoError(err) defer conn.Close() client := pb.NewDebugdClient(conn) - resp, err := client.UploadSystemServiceUnits(context.Background(), tc.request) + resp, err := client.UploadSystemServiceUnits(t.Context(), tc.request) grpcServ.GracefulStop() @@ -371,8 +371,8 @@ type netDialer interface { DialContext(_ context.Context, network, address string) (net.Conn, error) } -func dial(ctx context.Context, dialer netDialer, target string) (*grpc.ClientConn, error) { - return grpc.DialContext(ctx, target, +func dial(dialer netDialer, target string) (*grpc.ClientConn, error) { + return grpc.NewClient(target, grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { return dialer.DialContext(ctx, "tcp", addr) }), @@ -414,7 +414,7 @@ func setupServerWithConn(endpoint string, serv *debugdServer) (*grpc.Server, *gr lis := dialer.GetListener(endpoint) go grpcServ.Serve(lis) - conn, err := dial(context.Background(), dialer, endpoint) + conn, err := dial(dialer, endpoint) if err != nil { return nil, nil, err } diff --git a/debugd/internal/filetransfer/BUILD.bazel b/debugd/internal/filetransfer/BUILD.bazel index f59b0cb82..db82609db 100644 --- a/debugd/internal/filetransfer/BUILD.bazel +++ b/debugd/internal/filetransfer/BUILD.bazel @@ -13,8 +13,6 @@ go_library( "//debugd/internal/debugd", "//debugd/internal/filetransfer/streamer", "//debugd/service", - "//internal/logger", - "@org_uber_go_zap//:zap", ], ) diff --git a/debugd/internal/filetransfer/chunkstream.go b/debugd/internal/filetransfer/chunkstream.go index 9c36b968f..5fea59a15 100644 --- a/debugd/internal/filetransfer/chunkstream.go +++ b/debugd/internal/filetransfer/chunkstream.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package filetransfer diff --git a/debugd/internal/filetransfer/chunkstream_test.go b/debugd/internal/filetransfer/chunkstream_test.go index f01cbc136..e09e144de 100644 --- a/debugd/internal/filetransfer/chunkstream_test.go +++ b/debugd/internal/filetransfer/chunkstream_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package filetransfer diff --git a/debugd/internal/filetransfer/filetransfer.go b/debugd/internal/filetransfer/filetransfer.go index ff90bdf09..c15c5d0e4 100644 --- a/debugd/internal/filetransfer/filetransfer.go +++ b/debugd/internal/filetransfer/filetransfer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package filetransfer implements the exchange of files between cdgb <-> debugd @@ -10,16 +10,16 @@ package filetransfer import ( "errors" + "fmt" "io" "io/fs" + "log/slog" "sync" "sync/atomic" "github.com/edgelesssys/constellation/v2/debugd/internal/debugd" "github.com/edgelesssys/constellation/v2/debugd/internal/filetransfer/streamer" pb "github.com/edgelesssys/constellation/v2/debugd/service" - "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" ) // RecvFilesStream is a stream that receives FileTransferMessages. @@ -35,7 +35,7 @@ type SendFilesStream interface { // FileTransferer manages sending and receiving of files. type FileTransferer struct { fileMux sync.RWMutex - log *logger.Logger + log *slog.Logger receiveStarted bool receiveFinished atomic.Bool files []FileStat @@ -44,7 +44,7 @@ type FileTransferer struct { } // New creates a new FileTransferer. -func New(log *logger.Logger, streamer streamReadWriter, showProgress bool) *FileTransferer { +func New(log *slog.Logger, streamer streamReadWriter, showProgress bool) *FileTransferer { return &FileTransferer{ log: log, streamer: streamer, @@ -146,7 +146,7 @@ func (s *FileTransferer) handleFileRecv(stream RecvFilesStream) (bool, error) { if header == nil { return false, errors.New("first message must be a header message") } - s.log.Infof("Starting file receive of %q", header.TargetPath) + s.log.Info(fmt.Sprintf("Starting file receive of %q", header.TargetPath)) s.addFile(FileStat{ SourcePath: header.TargetPath, TargetPath: header.TargetPath, @@ -160,10 +160,10 @@ func (s *FileTransferer) handleFileRecv(stream RecvFilesStream) (bool, error) { }) recvChunkStream := &recvChunkStream{stream: stream} if err := s.streamer.WriteStream(header.TargetPath, recvChunkStream, s.showProgress); err != nil { - s.log.With(zap.Error(err)).Errorf("Receive of file %q failed", header.TargetPath) + s.log.With(slog.Any("error", err)).Error(fmt.Sprintf("Receive of file %q failed", header.TargetPath)) return false, err } - s.log.Infof("Finished file receive of %q", header.TargetPath) + s.log.Info(fmt.Sprintf("Finished file receive of %q", header.TargetPath)) return false, nil } diff --git a/debugd/internal/filetransfer/filetransfer_test.go b/debugd/internal/filetransfer/filetransfer_test.go index 73a977e74..7628edc3e 100644 --- a/debugd/internal/filetransfer/filetransfer_test.go +++ b/debugd/internal/filetransfer/filetransfer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package filetransfer @@ -20,7 +20,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestSendFiles(t *testing.T) { diff --git a/debugd/internal/filetransfer/streamer/streamer.go b/debugd/internal/filetransfer/streamer/streamer.go index 3648bb4d5..f48276ee9 100644 --- a/debugd/internal/filetransfer/streamer/streamer.go +++ b/debugd/internal/filetransfer/streamer/streamer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package streamer implements streaming of files over gRPC. diff --git a/debugd/internal/filetransfer/streamer/streamer_test.go b/debugd/internal/filetransfer/streamer/streamer_test.go index a1cd333dd..06e95324d 100644 --- a/debugd/internal/filetransfer/streamer/streamer_test.go +++ b/debugd/internal/filetransfer/streamer/streamer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package streamer @@ -19,7 +19,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestWriteStream(t *testing.T) { diff --git a/debugd/logstash/Dockerfile b/debugd/logstash/Dockerfile index 45bf35887..8d538596a 100644 --- a/debugd/logstash/Dockerfile +++ b/debugd/logstash/Dockerfile @@ -1,11 +1,11 @@ -FROM fedora:38@sha256:8285246bd5fad4e76e17a71c88dee34c49e2f227dab4ce7df704b592f8e72d41 AS build +FROM fedora:40@sha256:3c86d25fef9d2001712bc3d9b091fc40cf04be4767e48f1aa3b785bf58d300ed AS build ARG LOGSTASH_VER=8.6.1 RUN curl -fsSLO https://artifacts.opensearch.org/logstash/logstash-oss-with-opensearch-output-plugin-$LOGSTASH_VER-linux-x64.tar.gz RUN tar -zxvf logstash-oss-with-opensearch-output-plugin-$LOGSTASH_VER-linux-x64.tar.gz -FROM fedora:38@sha256:8285246bd5fad4e76e17a71c88dee34c49e2f227dab4ce7df704b592f8e72d41 AS release +FROM fedora:40@sha256:3c86d25fef9d2001712bc3d9b091fc40cf04be4767e48f1aa3b785bf58d300ed AS release COPY --from=build logstash-* /usr/share/logstash diff --git a/debugd/logstash/assets.go b/debugd/logstash/assets.go index e49e1f60d..4fda7bb88 100644 --- a/debugd/logstash/assets.go +++ b/debugd/logstash/assets.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package logstash diff --git a/debugd/metricbeat/Dockerfile b/debugd/metricbeat/Dockerfile index e4f84123a..11694af43 100644 --- a/debugd/metricbeat/Dockerfile +++ b/debugd/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:38@sha256:8285246bd5fad4e76e17a71c88dee34c49e2f227dab4ce7df704b592f8e72d41 AS release +FROM fedora:40@sha256:3c86d25fef9d2001712bc3d9b091fc40cf04be4767e48f1aa3b785bf58d300ed AS release RUN dnf install -y https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-8.9.2-x86_64.rpm diff --git a/debugd/metricbeat/assets.go b/debugd/metricbeat/assets.go index 8f3f954f1..faa3dc8fa 100644 --- a/debugd/metricbeat/assets.go +++ b/debugd/metricbeat/assets.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package metricbeat diff --git a/debugd/service/debugd.pb.go b/debugd/service/debugd.pb.go index 056a5b6c3..8414c895f 100644 --- a/debugd/service/debugd.pb.go +++ b/debugd/service/debugd.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v4.22.1 +// protoc-gen-go v1.36.6 +// protoc v5.29.1 // source: debugd/service/debugd.proto package service @@ -15,6 +15,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -172,20 +173,17 @@ func (UploadSystemdServiceUnitsStatus) EnumDescriptor() ([]byte, []int) { } type SetInfoRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Info []*Info `protobuf:"bytes,1,rep,name=info,proto3" json:"info,omitempty"` unknownFields protoimpl.UnknownFields - - Info []*Info `protobuf:"bytes,1,rep,name=info,proto3" json:"info,omitempty"` + sizeCache protoimpl.SizeCache } func (x *SetInfoRequest) Reset() { *x = SetInfoRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SetInfoRequest) String() string { @@ -196,7 +194,7 @@ func (*SetInfoRequest) ProtoMessage() {} func (x *SetInfoRequest) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -219,20 +217,17 @@ func (x *SetInfoRequest) GetInfo() []*Info { } type SetInfoResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Status SetInfoStatus `protobuf:"varint,1,opt,name=status,proto3,enum=debugd.SetInfoStatus" json:"status,omitempty"` unknownFields protoimpl.UnknownFields - - Status SetInfoStatus `protobuf:"varint,1,opt,name=status,proto3,enum=debugd.SetInfoStatus" json:"status,omitempty"` + sizeCache protoimpl.SizeCache } func (x *SetInfoResponse) Reset() { *x = SetInfoResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SetInfoResponse) String() string { @@ -243,7 +238,7 @@ func (*SetInfoResponse) ProtoMessage() {} func (x *SetInfoResponse) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -266,18 +261,16 @@ func (x *SetInfoResponse) GetStatus() SetInfoStatus { } type GetInfoRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetInfoRequest) Reset() { *x = GetInfoRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetInfoRequest) String() string { @@ -288,7 +281,7 @@ func (*GetInfoRequest) ProtoMessage() {} func (x *GetInfoRequest) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -304,20 +297,17 @@ func (*GetInfoRequest) Descriptor() ([]byte, []int) { } type GetInfoResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Info []*Info `protobuf:"bytes,1,rep,name=info,proto3" json:"info,omitempty"` unknownFields protoimpl.UnknownFields - - Info []*Info `protobuf:"bytes,1,rep,name=info,proto3" json:"info,omitempty"` + sizeCache protoimpl.SizeCache } func (x *GetInfoResponse) Reset() { *x = GetInfoResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetInfoResponse) String() string { @@ -328,7 +318,7 @@ func (*GetInfoResponse) ProtoMessage() {} func (x *GetInfoResponse) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -351,21 +341,18 @@ func (x *GetInfoResponse) GetInfo() []*Info { } type Info struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` unknownFields protoimpl.UnknownFields - - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + sizeCache protoimpl.SizeCache } func (x *Info) Reset() { *x = Info{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Info) String() string { @@ -376,7 +363,7 @@ func (*Info) ProtoMessage() {} func (x *Info) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -406,18 +393,16 @@ func (x *Info) GetValue() string { } type DownloadFilesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *DownloadFilesRequest) Reset() { *x = DownloadFilesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DownloadFilesRequest) String() string { @@ -428,7 +413,7 @@ func (*DownloadFilesRequest) ProtoMessage() {} func (x *DownloadFilesRequest) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -444,24 +429,21 @@ func (*DownloadFilesRequest) Descriptor() ([]byte, []int) { } type FileTransferMessage struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Kind: + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Kind: // // *FileTransferMessage_Header // *FileTransferMessage_Chunk - Kind isFileTransferMessage_Kind `protobuf_oneof:"kind"` + Kind isFileTransferMessage_Kind `protobuf_oneof:"kind"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *FileTransferMessage) Reset() { *x = FileTransferMessage{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *FileTransferMessage) String() string { @@ -472,7 +454,7 @@ func (*FileTransferMessage) ProtoMessage() {} func (x *FileTransferMessage) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -487,23 +469,27 @@ func (*FileTransferMessage) Descriptor() ([]byte, []int) { return file_debugd_service_debugd_proto_rawDescGZIP(), []int{6} } -func (m *FileTransferMessage) GetKind() isFileTransferMessage_Kind { - if m != nil { - return m.Kind +func (x *FileTransferMessage) GetKind() isFileTransferMessage_Kind { + if x != nil { + return x.Kind } return nil } func (x *FileTransferMessage) GetHeader() *FileTransferHeader { - if x, ok := x.GetKind().(*FileTransferMessage_Header); ok { - return x.Header + if x != nil { + if x, ok := x.Kind.(*FileTransferMessage_Header); ok { + return x.Header + } } return nil } func (x *FileTransferMessage) GetChunk() *Chunk { - if x, ok := x.GetKind().(*FileTransferMessage_Chunk); ok { - return x.Chunk + if x != nil { + if x, ok := x.Kind.(*FileTransferMessage_Chunk); ok { + return x.Chunk + } } return nil } @@ -525,22 +511,19 @@ func (*FileTransferMessage_Header) isFileTransferMessage_Kind() {} func (*FileTransferMessage_Chunk) isFileTransferMessage_Kind() {} type FileTransferHeader struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - TargetPath string `protobuf:"bytes,1,opt,name=targetPath,proto3" json:"targetPath,omitempty"` - Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` - OverrideServiceUnit *string `protobuf:"bytes,4,opt,name=overrideServiceUnit,proto3,oneof" json:"overrideServiceUnit,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + TargetPath string `protobuf:"bytes,1,opt,name=targetPath,proto3" json:"targetPath,omitempty"` + Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` + OverrideServiceUnit *string `protobuf:"bytes,4,opt,name=overrideServiceUnit,proto3,oneof" json:"overrideServiceUnit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *FileTransferHeader) Reset() { *x = FileTransferHeader{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *FileTransferHeader) String() string { @@ -551,7 +534,7 @@ func (*FileTransferHeader) ProtoMessage() {} func (x *FileTransferHeader) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -588,21 +571,18 @@ func (x *FileTransferHeader) GetOverrideServiceUnit() string { } type Chunk struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + Last bool `protobuf:"varint,2,opt,name=last,proto3" json:"last,omitempty"` unknownFields protoimpl.UnknownFields - - Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` - Last bool `protobuf:"varint,2,opt,name=last,proto3" json:"last,omitempty"` + sizeCache protoimpl.SizeCache } func (x *Chunk) Reset() { *x = Chunk{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Chunk) String() string { @@ -613,7 +593,7 @@ func (*Chunk) ProtoMessage() {} func (x *Chunk) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -643,20 +623,18 @@ func (x *Chunk) GetLast() bool { } type UploadFilesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Status UploadFilesStatus `protobuf:"varint,1,opt,name=status,proto3,enum=debugd.UploadFilesStatus" json:"status,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` unknownFields protoimpl.UnknownFields - - Status UploadFilesStatus `protobuf:"varint,1,opt,name=status,proto3,enum=debugd.UploadFilesStatus" json:"status,omitempty"` + sizeCache protoimpl.SizeCache } func (x *UploadFilesResponse) Reset() { *x = UploadFilesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *UploadFilesResponse) String() string { @@ -667,7 +645,7 @@ func (*UploadFilesResponse) ProtoMessage() {} func (x *UploadFilesResponse) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -689,22 +667,26 @@ func (x *UploadFilesResponse) GetStatus() UploadFilesStatus { return UploadFilesStatus_UPLOAD_FILES_SUCCESS } -type ServiceUnit struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *UploadFilesResponse) GetError() string { + if x != nil { + return x.Error + } + return "" +} - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Contents string `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"` +type ServiceUnit struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Contents string `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ServiceUnit) Reset() { *x = ServiceUnit{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ServiceUnit) String() string { @@ -715,7 +697,7 @@ func (*ServiceUnit) ProtoMessage() {} func (x *ServiceUnit) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -745,20 +727,17 @@ func (x *ServiceUnit) GetContents() string { } type UploadSystemdServiceUnitsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Units []*ServiceUnit `protobuf:"bytes,1,rep,name=units,proto3" json:"units,omitempty"` unknownFields protoimpl.UnknownFields - - Units []*ServiceUnit `protobuf:"bytes,1,rep,name=units,proto3" json:"units,omitempty"` + sizeCache protoimpl.SizeCache } func (x *UploadSystemdServiceUnitsRequest) Reset() { *x = UploadSystemdServiceUnitsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *UploadSystemdServiceUnitsRequest) String() string { @@ -769,7 +748,7 @@ func (*UploadSystemdServiceUnitsRequest) ProtoMessage() {} func (x *UploadSystemdServiceUnitsRequest) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -792,20 +771,17 @@ func (x *UploadSystemdServiceUnitsRequest) GetUnits() []*ServiceUnit { } type UploadSystemdServiceUnitsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Status UploadSystemdServiceUnitsStatus `protobuf:"varint,1,opt,name=status,proto3,enum=debugd.UploadSystemdServiceUnitsStatus" json:"status,omitempty"` unknownFields protoimpl.UnknownFields - - Status UploadSystemdServiceUnitsStatus `protobuf:"varint,1,opt,name=status,proto3,enum=debugd.UploadSystemdServiceUnitsStatus" json:"status,omitempty"` + sizeCache protoimpl.SizeCache } func (x *UploadSystemdServiceUnitsResponse) Reset() { *x = UploadSystemdServiceUnitsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_debugd_service_debugd_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_debugd_service_debugd_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *UploadSystemdServiceUnitsResponse) String() string { @@ -816,7 +792,7 @@ func (*UploadSystemdServiceUnitsResponse) ProtoMessage() {} func (x *UploadSystemdServiceUnitsResponse) ProtoReflect() protoreflect.Message { mi := &file_debugd_service_debugd_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -840,136 +816,78 @@ func (x *UploadSystemdServiceUnitsResponse) GetStatus() UploadSystemdServiceUnit var File_debugd_service_debugd_proto protoreflect.FileDescriptor -var file_debugd_service_debugd_proto_rawDesc = []byte{ - 0x0a, 0x1b, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x2f, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x64, - 0x65, 0x62, 0x75, 0x67, 0x64, 0x22, 0x32, 0x0a, 0x0e, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x40, 0x0a, 0x0f, 0x53, 0x65, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x64, - 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x10, 0x0a, 0x0e, 0x47, - 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x33, 0x0a, - 0x0f, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x20, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, - 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, - 0x66, 0x6f, 0x22, 0x2e, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x7a, 0x0a, 0x13, 0x46, 0x69, - 0x6c, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x34, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, - 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, - 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x05, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x42, 0x06, - 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x97, 0x01, 0x0a, 0x12, 0x46, 0x69, 0x6c, 0x65, 0x54, - 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x1e, 0x0a, - 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, - 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, - 0x65, 0x12, 0x35, 0x0a, 0x13, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, - 0x52, 0x13, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x55, 0x6e, 0x69, 0x74, 0x88, 0x01, 0x01, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x6f, 0x76, 0x65, - 0x72, 0x72, 0x69, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, - 0x22, 0x35, 0x0a, 0x05, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x22, 0x48, 0x0a, 0x13, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, - 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, - 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, - 0x6c, 0x65, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x22, 0x3d, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, - 0x22, 0x4d, 0x0a, 0x20, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x52, 0x05, 0x75, 0x6e, 0x69, 0x74, 0x73, 0x22, - 0x64, 0x0a, 0x21, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, - 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x2a, 0x3f, 0x0a, 0x0d, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x45, 0x54, 0x5f, 0x49, 0x4e, - 0x46, 0x4f, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, - 0x53, 0x45, 0x54, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, - 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01, 0x2a, 0xb1, 0x01, 0x0a, 0x11, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x14, - 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x5f, 0x53, 0x55, 0x43, - 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, - 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x5f, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x46, 0x41, - 0x49, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, - 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x5f, 0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, 0x5f, 0x53, - 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, 0x55, 0x50, 0x4c, 0x4f, - 0x41, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x5f, 0x41, 0x4c, 0x52, 0x45, 0x41, 0x44, 0x59, - 0x5f, 0x46, 0x49, 0x4e, 0x49, 0x53, 0x48, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1d, 0x0a, 0x19, 0x55, - 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x5f, 0x53, 0x54, 0x41, 0x52, - 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x2a, 0x75, 0x0a, 0x1f, 0x55, 0x70, - 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x0a, - 0x24, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x44, 0x5f, - 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54, 0x53, 0x5f, 0x53, 0x55, - 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x28, 0x0a, 0x24, 0x55, 0x50, 0x4c, 0x4f, 0x41, - 0x44, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x44, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, - 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, - 0x01, 0x32, 0x94, 0x03, 0x0a, 0x06, 0x44, 0x65, 0x62, 0x75, 0x67, 0x64, 0x12, 0x3c, 0x0a, 0x07, - 0x53, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, - 0x2e, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x17, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x07, 0x47, 0x65, - 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x47, - 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, - 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0b, 0x55, 0x70, 0x6c, 0x6f, - 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1b, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, - 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x4e, 0x0a, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, - 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x46, 0x69, - 0x6c, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71, 0x0a, 0x18, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, - 0x73, 0x12, 0x28, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, - 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x64, 0x65, - 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, - 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x2f, 0x76, 0x32, 0x2f, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_debugd_service_debugd_proto_rawDesc = "" + + "\n" + + "\x1bdebugd/service/debugd.proto\x12\x06debugd\"2\n" + + "\x0eSetInfoRequest\x12 \n" + + "\x04info\x18\x01 \x03(\v2\f.debugd.InfoR\x04info\"@\n" + + "\x0fSetInfoResponse\x12-\n" + + "\x06status\x18\x01 \x01(\x0e2\x15.debugd.SetInfoStatusR\x06status\"\x10\n" + + "\x0eGetInfoRequest\"3\n" + + "\x0fGetInfoResponse\x12 \n" + + "\x04info\x18\x01 \x03(\v2\f.debugd.InfoR\x04info\".\n" + + "\x04Info\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"\x16\n" + + "\x14DownloadFilesRequest\"z\n" + + "\x13FileTransferMessage\x124\n" + + "\x06header\x18\x01 \x01(\v2\x1a.debugd.FileTransferHeaderH\x00R\x06header\x12%\n" + + "\x05chunk\x18\x02 \x01(\v2\r.debugd.ChunkH\x00R\x05chunkB\x06\n" + + "\x04kind\"\x97\x01\n" + + "\x12FileTransferHeader\x12\x1e\n" + + "\n" + + "targetPath\x18\x01 \x01(\tR\n" + + "targetPath\x12\x12\n" + + "\x04mode\x18\x03 \x01(\rR\x04mode\x125\n" + + "\x13overrideServiceUnit\x18\x04 \x01(\tH\x00R\x13overrideServiceUnit\x88\x01\x01B\x16\n" + + "\x14_overrideServiceUnit\"5\n" + + "\x05Chunk\x12\x18\n" + + "\acontent\x18\x01 \x01(\fR\acontent\x12\x12\n" + + "\x04last\x18\x02 \x01(\bR\x04last\"^\n" + + "\x13UploadFilesResponse\x121\n" + + "\x06status\x18\x01 \x01(\x0e2\x19.debugd.UploadFilesStatusR\x06status\x12\x14\n" + + "\x05error\x18\x02 \x01(\tR\x05error\"=\n" + + "\vServiceUnit\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" + + "\bcontents\x18\x02 \x01(\tR\bcontents\"M\n" + + " UploadSystemdServiceUnitsRequest\x12)\n" + + "\x05units\x18\x01 \x03(\v2\x13.debugd.ServiceUnitR\x05units\"d\n" + + "!UploadSystemdServiceUnitsResponse\x12?\n" + + "\x06status\x18\x01 \x01(\x0e2'.debugd.UploadSystemdServiceUnitsStatusR\x06status*?\n" + + "\rSetInfoStatus\x12\x14\n" + + "\x10SET_INFO_SUCCESS\x10\x00\x12\x18\n" + + "\x14SET_INFO_ALREADY_SET\x10\x01*\xb1\x01\n" + + "\x11UploadFilesStatus\x12\x18\n" + + "\x14UPLOAD_FILES_SUCCESS\x10\x00\x12\x1e\n" + + "\x1aUPLOAD_FILES_UPLOAD_FAILED\x10\x01\x12 \n" + + "\x1cUPLOAD_FILES_ALREADY_STARTED\x10\x02\x12!\n" + + "\x1dUPLOAD_FILES_ALREADY_FINISHED\x10\x03\x12\x1d\n" + + "\x19UPLOAD_FILES_START_FAILED\x10\x04*u\n" + + "\x1fUploadSystemdServiceUnitsStatus\x12(\n" + + "$UPLOAD_SYSTEMD_SERVICE_UNITS_SUCCESS\x10\x00\x12(\n" + + "$UPLOAD_SYSTEMD_SERVICE_UNITS_FAILURE\x10\x012\x94\x03\n" + + "\x06Debugd\x12<\n" + + "\aSetInfo\x12\x16.debugd.SetInfoRequest\x1a\x17.debugd.SetInfoResponse\"\x00\x12<\n" + + "\aGetInfo\x12\x16.debugd.GetInfoRequest\x1a\x17.debugd.GetInfoResponse\"\x00\x12K\n" + + "\vUploadFiles\x12\x1b.debugd.FileTransferMessage\x1a\x1b.debugd.UploadFilesResponse\"\x00(\x01\x12N\n" + + "\rDownloadFiles\x12\x1c.debugd.DownloadFilesRequest\x1a\x1b.debugd.FileTransferMessage\"\x000\x01\x12q\n" + + "\x18UploadSystemServiceUnits\x12(.debugd.UploadSystemdServiceUnitsRequest\x1a).debugd.UploadSystemdServiceUnitsResponse\"\x00B8Z6github.com/edgelesssys/constellation/v2/debugd/serviceb\x06proto3" var ( file_debugd_service_debugd_proto_rawDescOnce sync.Once - file_debugd_service_debugd_proto_rawDescData = file_debugd_service_debugd_proto_rawDesc + file_debugd_service_debugd_proto_rawDescData []byte ) func file_debugd_service_debugd_proto_rawDescGZIP() []byte { file_debugd_service_debugd_proto_rawDescOnce.Do(func() { - file_debugd_service_debugd_proto_rawDescData = protoimpl.X.CompressGZIP(file_debugd_service_debugd_proto_rawDescData) + file_debugd_service_debugd_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_debugd_service_debugd_proto_rawDesc), len(file_debugd_service_debugd_proto_rawDesc))) }) return file_debugd_service_debugd_proto_rawDescData } var file_debugd_service_debugd_proto_enumTypes = make([]protoimpl.EnumInfo, 3) var file_debugd_service_debugd_proto_msgTypes = make([]protoimpl.MessageInfo, 13) -var file_debugd_service_debugd_proto_goTypes = []interface{}{ +var file_debugd_service_debugd_proto_goTypes = []any{ (SetInfoStatus)(0), // 0: debugd.SetInfoStatus (UploadFilesStatus)(0), // 1: debugd.UploadFilesStatus (UploadSystemdServiceUnitsStatus)(0), // 2: debugd.UploadSystemdServiceUnitsStatus @@ -1018,174 +936,16 @@ func file_debugd_service_debugd_proto_init() { if File_debugd_service_debugd_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_debugd_service_debugd_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetInfoRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SetInfoResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetInfoRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetInfoResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Info); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DownloadFilesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileTransferMessage); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileTransferHeader); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Chunk); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UploadFilesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ServiceUnit); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UploadSystemdServiceUnitsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_debugd_service_debugd_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UploadSystemdServiceUnitsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_debugd_service_debugd_proto_msgTypes[6].OneofWrappers = []interface{}{ + file_debugd_service_debugd_proto_msgTypes[6].OneofWrappers = []any{ (*FileTransferMessage_Header)(nil), (*FileTransferMessage_Chunk)(nil), } - file_debugd_service_debugd_proto_msgTypes[7].OneofWrappers = []interface{}{} + file_debugd_service_debugd_proto_msgTypes[7].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_debugd_service_debugd_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_debugd_service_debugd_proto_rawDesc), len(file_debugd_service_debugd_proto_rawDesc)), NumEnums: 3, NumMessages: 13, NumExtensions: 0, @@ -1197,7 +957,6 @@ func file_debugd_service_debugd_proto_init() { MessageInfos: file_debugd_service_debugd_proto_msgTypes, }.Build() File_debugd_service_debugd_proto = out.File - file_debugd_service_debugd_proto_rawDesc = nil file_debugd_service_debugd_proto_goTypes = nil file_debugd_service_debugd_proto_depIdxs = nil } diff --git a/debugd/service/debugd.proto b/debugd/service/debugd.proto index db9c3b896..77bc94a60 100644 --- a/debugd/service/debugd.proto +++ b/debugd/service/debugd.proto @@ -58,6 +58,7 @@ message Chunk { message UploadFilesResponse { UploadFilesStatus status = 1; + string error = 2; } enum UploadFilesStatus { diff --git a/dev-docs/chain-of-trust.jpg b/dev-docs/chain-of-trust.jpg new file mode 100644 index 000000000..713e4715e Binary files /dev/null and b/dev-docs/chain-of-trust.jpg differ diff --git a/dev-docs/conventions.md b/dev-docs/conventions.md index 26f8ba3fb..5f9e9c132 100644 --- a/dev-docs/conventions.md +++ b/dev-docs/conventions.md @@ -14,64 +14,74 @@ This project also aims to follow the [Go Proverbs](https://go-proverbs.github.io ## Linting This projects uses [golangci-lint](https://golangci-lint.run/) for linting. -You can [install golangci-lint](https://golangci-lint.run/usage/install/#linux-and-windows) locally, +You can [install golangci-lint](https://golangci-lint.run/welcome/install/#local-installation) locally, but there is also a CI action to ensure compliance. -It is also recommended to use golangci-lint (and [gofumpt](https://github.com/mvdan/gofumpt) as formatter) in your IDE, by adding the recommended VS Code Settings or by [configuring it yourself](https://golangci-lint.run/usage/integrations/#editor-integration) +It is also recommended to use golangci-lint (and [gofumpt](https://github.com/mvdan/gofumpt) as formatter) in your IDE, by adding the recommended VS Code Settings or by [configuring it yourself](https://golangci-lint.run/welcome/integrations/) ## Logging -We use a [custom subset](/internal/logger/) of [zap](https://pkg.go.dev/go.uber.org/zap) to provide logging for Constellation’s services and components. +We use [slog](https://pkg.go.dev/log/slog) for logging. Usage instructions can be found in the package documentation. -Certain components may further specify a subset of the logger for their use. For example, the CLI has a debug-only logger, restricting the use of the logger to only `Debugf()`. +Certain components may further specify a subset of the logger for their use. For example, the CLI has a debug-only logger, restricting the use of the logger to only `Debug()`. Further we try to adhere to the following guidelines: * Do not log potentially sensitive information, e.g. variables that contain keys, secrets or otherwise protected information. +* Create a text or JSON logger using the helper functions in the `logger` package. These create a `slog.Logger` with useful defaults. + + Example: + + ```Go + log := logger.NewTextLogger(slog.LevelDebug) + log.Debug("A debug message") + ``` + * Start log messages in uppercase and end without a punctuation mark. Exclamation, question marks, or ellipsis may be used where appropriate. Example: ```Go - log.Infof("This is a log message") - log.Infof("Waiting to do something...") + log.Info("This is a log message") + log.Info("Waiting to do something...") log.Error("A critical error occurred!") ``` -* Use the `With()` method to add structured context to your log messages. The context tags should be easily searchable to allow for easy log filtering. Try to keep consistent tag naming! +* Use additional arguments to add structured context to your log messages. The context tags should be easily searchable to allow for easy log filtering. Try to keep consistent tag naming! Example: ```Go - log.With(zap.Error(someError), zap.String("ip", "192.0.2.1")).Errorf("Connecting to IP failed") + log.Error("Connecting to IP failed", "error", someError, "ip", "192.0.2.1") ``` -* Log messages may use format strings to produce human readable messages. However, the information should also be present as structured context fields if it might be valuable for debugging purposes. - - Example: +* Log messages may use format strings to produce human readable messages. However, the information should also be present as structured context fields if it might be valuable for debugging purposes. So, instead of writing ```Go - log.Infof("Starting server on %s:%s", addr, port) + log.Info(fmt.Sprintf("Starting server on %s:%s", addr, port)) ``` -* Usage of the `Fatalf()` method should be constrained to the main package of an application only! + You should write + + ```Go + log.Info("Starting server", "addr", addr, "port", port) + ``` * Use log levels to configure how detailed the logs of you application should be. - * `Debugf()` for log low level and detailed information. This may include variable dumps, but should not disclose sensitive information, e.g. keys or secret tokens. - * `Infof()` for general information. - * `Warnf()` for information that may indicate unwanted behavior, but is not an application error. Commonly used by retry loops. - * `Errorf()` to log information about any errors that occurred. - * `Fatalf()` to log information about any errors that occurred and then exit the program. Should only be used in the main package of an application. + * `Debug()` for log low level and detailed information. This may include variable dumps, but should not disclose sensitive information, e.g. keys or secret tokens. + * `Info()` for general information. + * `Warn()` for information that may indicate unwanted behavior, but is not an application error. Commonly used by retry loops. + * `Error()` to log information about any errors that occurred. -* Loggers passed to subpackages of an application may use the `Named()` method for better understanding of where a message originated. +* Loggers passed to subpackages of an application may use the `WithGroup()` method for better understanding of where a message originated. Example: ```Go - grpcServer, err := server.New(log.Named("server")) + grpcServer, err := server.New(log.WithGroup("server")) ``` ## Nested Go modules diff --git a/dev-docs/howto/bare-metal/README.md b/dev-docs/howto/bare-metal/README.md new file mode 100644 index 000000000..e4df4cd97 --- /dev/null +++ b/dev-docs/howto/bare-metal/README.md @@ -0,0 +1,66 @@ +# Bare-metal SNP setup for Constellation + +## Prepare Host + +The bare-metal host machine needs to be able to start SEV-SNP VMs. +A thorough explanation can be found here: . + +First checkout the snp-latest branch: + +```bash +git clone https://github.com/AMDESE/AMDSEV.git +cd AMDSEV +git checkout snp-latest +``` + +Then enable TPM2 support by setting `-DTPM2_ENABLE` in the OVMF build command +found in `common.sh`: + +```patch +diff --git a/common.sh b/common.sh +index 9eee947..52bf507 100755 +--- a/common.sh ++++ b/common.sh +@@ -155,7 +155,7 @@ build_install_ovmf() + GCCVERS="GCC5" + fi + +- BUILD_CMD="nice build -q --cmd-len=64436 -DDEBUG_ON_SERIAL_PORT=TRUE -n $(getconf _NPROCESSORS_ONLN) ${GCCVERS:+-t $GCCVERS} -a X64 -p OvmfPkg/OvmfPkgX64.dsc" ++ BUILD_CMD="nice build -q --cmd-len=64436 -DTPM2_ENABLE -DDEBUG_ON_SERIAL_PORT=TRUE -n $(getconf _NPROCESSORS_ONLN) ${GCCVERS:+-t $GCCVERS} -a X64 -p OvmfPkg/OvmfPkgX64.dsc" + + # initialize git repo, or update existing remote to currently configured one + if [ -d ovmf ]; then +``` + +Build and package the binaries. Then install the newly build kernel: + +```bash +./build.sh --package +cd linux +dpkg -i linux-image-6.9.0-rc7-snp-host-05b10142ac6a_6.9.0-rc7-g05b10142ac6a-2_amd64.deb +``` + +Reboot, verify that the right BIOS setting are set as described in + +and select the new kernel in the boot menu. Note that GRUB usually automatically +select the newest installed kernel as default. + +Download a Constellation qemu image, the `constellation-conf.yaml`, and +the `launch-constellation.sh` script in the directory right next to the +`AMDSEV` folder. + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/dev-docs/howto/bare-metal/launch-constellation.sh +wget https://cdn.confidential.cloud/constellation/v1/ref/main/stream/console/v2.17.0-pre.0.20240516182331-5fb2a2cb89f2/image/csp/qemu/qemu-vtpm/image.raw +wget < link to the constellation CLI provided by Edgeless > +wget < link to the constellation config provided by Edgeless > +``` + +Install and setup [docker](https://docs.docker.com/engine/install/), +install swtpm, dnsmasq and tmux. + +Then simply run: + +```bash +sudo ./launch-constellation.sh +``` diff --git a/dev-docs/howto/bare-metal/launch-constellation.sh b/dev-docs/howto/bare-metal/launch-constellation.sh new file mode 100644 index 000000000..27fc4f83b --- /dev/null +++ b/dev-docs/howto/bare-metal/launch-constellation.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash + +set -euo pipefail + +set -x + +function cleanup { + kill -SIGTERM "$(cat "${PWD}"/qemu-dnsmasq-br0.pid)" || true + rm "${PWD}"/qemu-dnsmasq-br0.pid || true + + kill -SIGTERM "$(cat "${PWD}"/swtpm0.pid)" || true + kill -SIGTERM "$(cat "${PWD}"/swtpm1.pid)" || true + + ip l delete br0 || true + ip l delete tap0 || true + ip l delete tap1 || true + + rm -r "${PWD}"/tpm0 || true + rm -r "${PWD}"/tpm1 || true + + rm OVMF_VARS_0.fd || true + rm OVMF_VARS_1.fd || true + + rm dnsmasq.leases || true + rm dnsmasq.log || true + + rm constellation-mastersecret.json || true + rm constellation-admin.conf || true + rm constellation-cluster.log || true + rm constellation-debug.log || true + rm constellation-state.yaml || true + rm -r constellation-upgrade || true + + docker stop metadata-server || true +} + +trap cleanup EXIT + +get_mac() { + printf '52:54:%02X:%02X:%02X:%02X' $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)) +} + +mac_0=$(get_mac) +mac_1=$(get_mac) + +# Regarding network setup see: https://bbs.archlinux.org/viewtopic.php?id=207907 + +dd if=/dev/zero of=disk0.img iflag=fullblock bs=1M count=10000 && sync +dd if=/dev/zero of=disk1.img iflag=fullblock bs=1M count=10000 && sync + +DEFAULT_INTERFACE=$(ip r show default | cut -d' ' -f5) + +ip link add name br0 type bridge || true +ip addr add 10.42.0.1/16 dev br0 || true +ip link set br0 up + +dnsmasq \ + --pid-file="${PWD}"/qemu-dnsmasq-br0.pid \ + --interface=br0 \ + --bind-interfaces \ + --log-facility="${PWD}"/dnsmasq.log \ + --dhcp-range=10.42.0.2,10.42.255.254 \ + --dhcp-leasefile="${PWD}"/dnsmasq.leases \ + --dhcp-host="${mac_0}",10.42.1.1,control-plane0 \ + --dhcp-host="${mac_1}",10.42.2.1,worker0 + +password=$(tr -dc 'A-Za-z0-9!?%=' < /dev/urandom | head -c 32) || true +password_hex=$(echo -n "${password}" | xxd -p -u -c 256) +echo "${password_hex}" + +# htpasswd from apache2-utils +password_bcrypt=$(htpasswd -bnBC 10 "" "${password}" | tr -d ':\n') + +docker run \ + -dit \ + --rm \ + --name metadata-server \ + --net=host \ + --mount type=bind,source="$(pwd)"/dnsmasq.leases,target=/dnsmasq.leases \ + ghcr.io/edgelesssys/constellation/qemu-metadata-api:v2.17.0-pre.0.20240603111213-d7ce6af383f2 \ + --dnsmasq-leases /dnsmasq.leases --initsecrethash "${password_bcrypt}" + +cat > ./constellation-state.yaml <<- EOM +version: v1 # Schema version of this state file. +# State of the cluster's cloud resources. These values are retrieved during +infrastructure: + uid: qemu # Unique identifier the cluster's cloud resources are tagged with. + clusterEndpoint: 10.42.1.1 # Endpoint the cluster can be reached at. This is the endpoint that is being used by the CLI. + inClusterEndpoint: 10.42.1.1 # The Cluster uses to reach itself. This might differ from the ClusterEndpoint in case e.g., + initSecret: "${password_hex}" # Secret used to authenticate the bootstrapping node. + # List of Subject Alternative Names (SANs) to add to the Kubernetes API server certificate. + apiServerCertSANs: + - 10.42.1.1 + name: mini-qemu # Name used in the cluster's named resources. + ipCidrNode: 10.42.0.0/16 # CIDR range of the cluster's nodes. +# DO NOT EDIT. State of the Constellation Kubernetes cluster. +clusterValues: + clusterID: "" # Unique identifier of the cluster. + ownerID: "" # Unique identifier of the owner of the cluster. + measurementSalt: "" # Salt used to generate the ClusterID on the bootstrapping node. +EOM + +sysctl net.ipv4.ip_forward=1 +sysctl net.ipv6.conf.default.forwarding=1 +sysctl net.ipv6.conf.all.forwarding=1 + +iptables -t nat -C POSTROUTING -o "${DEFAULT_INTERFACE}" -j MASQUERADE || iptables -t nat -I POSTROUTING -o "${DEFAULT_INTERFACE}" -j MASQUERADE +iptables -C FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT || iptables -I FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +iptables -P FORWARD ACCEPT + +ip tuntap add dev tap0 mode tap user "${USER}" || true +ip link set tap0 up promisc on +ip link set tap0 master br0 + +iptables -C FORWARD -i tap0 -o "${DEFAULT_INTERFACE}" -j ACCEPT || iptables -I FORWARD -i tap0 -o "${DEFAULT_INTERFACE}" -j ACCEPT + +ip tuntap add dev tap1 mode tap user "${USER}" || true +ip link set tap1 up promisc on +ip link set tap1 master br0 + +iptables -C FORWARD -i tap1 -o "${DEFAULT_INTERFACE}" -j ACCEPT || iptables -I FORWARD -i tap1 -o "${DEFAULT_INTERFACE}" -j ACCEPT + +# +# ovmf +# + +cp AMDSEV/usr/local/share/qemu/OVMF_VARS.fd OVMF_VARS_0.fd +cp AMDSEV/usr/local/share/qemu/OVMF_VARS.fd OVMF_VARS_1.fd + +# +# swtpm +# + +mkdir "${PWD}"/tpm0 || true +swtpm_setup --tpm2 --tpmstate "${PWD}/tpm0" --create-ek-cert --create-platform-cert --allow-signing --overwrite --pcr-banks - --logfile "${PWD}/tpm0/setup.log" +swtpm socket --tpm2 --tpmstate dir="${PWD}/tpm0",mode=0600 --ctrl type=unixio,path="${PWD}/tpm0/swtpm-sock" --log file="${PWD}/tpm0/tpm.log",level=20,truncate --pid file="${PWD}/swtpm0.pid" & + +mkdir "${PWD}"/tpm1 || true +swtpm_setup --tpm2 --tpmstate "${PWD}/tpm1" --create-ek-cert --create-platform-cert --allow-signing --overwrite --pcr-banks - --logfile "${PWD}/tpm1/setup.log" +swtpm socket --tpm2 --tpmstate dir="${PWD}/tpm1",mode=0600 --ctrl type=unixio,path="${PWD}/tpm1/swtpm-sock" --log file="${PWD}/tpm1/tpm.log",level=20,truncate --pid file="${PWD}/swtpm1.pid" & + +tmux new-session -d -s const-sess + +tmux split-window +tmux split-window + +launch_cmd_base_sev="AMDSEV/usr/local/bin/qemu-system-x86_64 \ + -enable-kvm \ + -cpu EPYC-v4 \ + -machine q35,smm=off \ + -smp 4,maxcpus=255 \ + -m 2048M,slots=5,maxmem=$((2048 + 8192))M \ + -no-reboot \ + -bios AMDSEV/usr/local/share/qemu/OVMF_CODE.fd \ + -drive file=./image.raw,if=none,id=disk1,format=raw,readonly=on \ + -device virtio-blk-pci,drive=disk1,id=virtio-disk1,disable-legacy=on,iommu_platform=true,bootindex=1 \ + -machine memory-encryption=sev0,vmport=off \ + -object memory-backend-memfd,id=ram1,size=2048M,share=true,prealloc=false \ + -machine memory-backend=ram1 \ + -object sev-snp-guest,id=sev0,cbitpos=51,reduced-phys-bits=1 \ + -nographic \ + -device virtio-blk-pci,drive=disk2,id=virtio-disk2 \ + -tpmdev emulator,id=tpm0,chardev=chrtpm \ + -device tpm-crb,tpmdev=tpm0" + +# shellcheck disable=2034 +launch_cmd_base_no_sev="AMDSEV/usr/local/bin/qemu-system-x86_64 \ + -enable-kvm \ + -cpu EPYC-v4 \ + -machine q35 \ + -smp 1,maxcpus=255 \ + -m 2048M,slots=5,maxmem=10240M \ + -no-reboot \ + -drive if=pflash,format=raw,unit=0,file=${PWD}/OVMF_CODE.fd,readonly=true \ + -drive file=./image.raw,if=none,id=disk1,format=raw,readonly=on \ + -device virtio-blk-pci,drive=disk1,id=virtio-disk1,disable-legacy=on,iommu_platform=true,bootindex=1 \ + -nographic \ + -device virtio-blk-pci,drive=disk2,id=virtio-disk2 \ + -tpmdev emulator,id=tpm0,chardev=chrtpm \ + -device tpm-crb,tpmdev=tpm0" + +launch_cmd_0="${launch_cmd_base_sev} \ + -drive if=pflash,format=raw,unit=0,file=${PWD}/OVMF_VARS_0.fd \ + -device virtio-net,netdev=network0,mac=${mac_0} \ + -netdev tap,id=network0,ifname=tap0,script=no,downscript=no \ + -drive file=./disk0.img,id=disk2,if=none,format=raw \ + -chardev socket,id=chrtpm,path=${PWD}/tpm0/swtpm-sock" +launch_cmd_1="${launch_cmd_base_sev} \ + -drive if=pflash,format=raw,unit=0,file=${PWD}/OVMF_VARS_1.fd \ + -device virtio-net,netdev=network0,mac=${mac_1} \ + -netdev tap,id=network0,ifname=tap1,script=no,downscript=no \ + -drive file=./disk1.img,id=disk2,if=none,format=raw \ + -chardev socket,id=chrtpm,path=${PWD}/tpm1/swtpm-sock" + +init_cmd="./constellation apply --skip-phases infrastructure" + +tmux send -t const-sess:0.0 "${launch_cmd_0}" ENTER +sleep 3 +tmux send -t const-sess:0.1 "${launch_cmd_1}" ENTER +tmux send -t const-sess:0.2 "${init_cmd}" ENTER + +tmux a -t const-sess diff --git a/dev-docs/howto/kubeconfigs.md b/dev-docs/howto/kubeconfigs.md new file mode 100644 index 000000000..0324e9bdc --- /dev/null +++ b/dev-docs/howto/kubeconfigs.md @@ -0,0 +1,88 @@ +# How to create kubeconfigs for users + +One of the first things to do after setting up a Constellation cluster is to hand out kubeconfig files to its prospective users. +Adhering to the *principle of least privilege*, it is not advisable to share the admin config with all cluster users. +Instead, users should authenticate individually to the API server, and permissions should be controlled by [RBAC]. + +Constellation users authenticate to the API server with a client TLS certificate, signed by the Kubernetes CA. +The user's identity and group memberships are taken from the certificates common name and organizations, respectively. +Details can be found in the upstream [authn documentation]. + +The [`kubeadm` documentation] describes a process for creating new kubeconfigs, but the instructions requires access to a control-plane node, or at least the Kubernetes CA certificate and key. +While the certificates can be extracted, e.g. by spawning a [node debugger pod], we can take a safer road that only requires `kubectl`. +The example script below creates a new kubeconfig for a user and optional group memberships. +It uses the [Kubernetes certificate API] to obtain a user certificate signed by the cluster CA. + +[RBAC]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +[authn documentation]: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes +[`kubeadm` documentation]: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/#kubeconfig-additional-users +[node debugger pod]: https://kubernetes.io/docs/tasks/debug/debug-cluster/kubectl-node-debug/ +[Kubernetes certificate API]: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + +```sh +#!/bin/sh +set -eu + +if [ $# -lt 2 ]; then + echo "Usage: $0 username [groupname...]" >&2 + exit 1 +fi + +user=$1 +shift + +subj="/CN=${user}" +for g in "$@"; do + subj="${subj}/O=$g" +done + +openssl req -newkey rsa:4096 -out ${user}.csr -keyout ${user}.key -nodes -subj "${subj}" + +kubectl apply -f - <${user}.pem +kubectl delete csr ${user} + +kubectl get cm kube-root-ca.crt -o go-template='{{ index .data "ca.crt" }}' >ca.pem +kubectl get cm kubeadm-config -n kube-system -o=jsonpath="{.data.ClusterConfiguration}" >clusterconfig.yaml +cluster=$(yq .clusterName clusterconfig.yaml) +endpoint=$(yq .controlPlaneEndpoint clusterconfig.yaml) + +cat >${user}.conf <= v2.14.0 +* A publicly routable VPN endpoint on premises that supports IPSec in IKEv2 + tunnel mode with NAT traversal enabled. +* A list of on-prem CIDRs that should be reachable from Constellation. + +## Setup + +1. Configure Cilium to route services for the VPN (see [Architecture](#architecture) for details). + * Edit the Cilium config: `kubectl -n kube-system edit configmap cilium-config`. + * Set the config item `enable-sctp: "true"`. + * Restart the Cilium agents: `kubectl -n kube-system rollout restart daemonset/cilium`. + +2. Create the Constellation VPN configuration file. ```sh helm inspect values . >config.yaml ``` -2. Install the Helm chart. +3. Populate the Constellation VPN configuration file. At least the following + need to be configured: + * The list of on-prem CIDRs (`peerCIDRs`). + * The `ipsec` subsection. + +4. Install the Helm chart. ```sh helm install -f config.yaml vpn . ``` -3. Follow the post-installation instructions displayed by the CLI. +5. Configure the on-prem gateway with Constellation's pod and service CIDR + (see `config.yaml`). + +## Things to try + +Ask CoreDNS about its own service IP: + +```sh +dig +notcp @10.96.0.10 kube-dns.kube-system.svc.cluster.local +``` + +Ask the Kubernetes API server about its wellbeing: + +```sh +curl --insecure https://10.96.0.1:6443/healthz +``` + +Ping a pod: + +```sh +ping $(kubectl get pods vpn-frontend-0 -o go-template --template '{{ .status.podIP }}') +``` ## Architecture -The VPN server is deployed as a `StatefulSet` to the cluster. It hosts the VPN frontend component, which is responsible for relaying traffic between the pod and the on-prem network, and the routing components that provide access to Constellation resources. The frontend supports IPSec and Wireguard. +The VPN server is deployed as a `StatefulSet` to the cluster. It hosts the VPN +frontend component, which is responsible for relaying traffic between the pod +and the on-prem network over an IPSec tunnel. -The VPN frontend is exposed with a public LoadBalancer to be accessible from the on-prem network. Traffic that reaches the VPN server pod is split into two categories: pod IPs and service IPs. +The VPN frontend is exposed with a public LoadBalancer so that it becomes +accessible from the on-prem network. -The pod IP range is NATed with an iptables rule. On-prem worklaods can establish connections to a pod IP, but the Constellation workloads will see the client IP translated to that of the VPN frontend pod. +An init container sets up IP routes on the frontend host and inside the +frontend pod. All routes are bound to the frontend pod's lxc interface and thus +deleted together with it. -The service IP range is handed to a transparent proxy running in the VPN frontend pod, which relays the connection to a backend pod. This is necessary because of the load-balancing mechanism of Cilium, which assumes service IP traffic to originate from the Constellation cluster itself. As for pod IP ranges, Constellation pods will only see the translated client address. +A VPN operator deployment is added that configures the `CiliumEndpoint` with +on-prem IP ranges, thus configuring routes on non-frontend hosts. The endpoint +shares the frontend pod's lifecycle. + +In Cilium's default configuration, service endpoints are resolved in cgroup +eBPF hooks that are not applicable to VPN traffic. We force Cilium to apply +service NAT at the LXC interface by enabling SCTP support. ## Limitations -* Service IPs need to be proxied by the VPN frontend pod. This is a single point of failure, and it may become a bottleneck. -* IPs are NATed, so the Constellation pods won't see the real on-prem IPs. -* NetworkPolicy can't be applied selectively to the on-prem ranges. -* No connectivity from Constellation to on-prem workloads. +* VPN traffic is handled by a single pod, which may become a bottleneck. +* Frontend pod restarts / migrations invalidate IPSec connections. +* Only pre-shared key authentication is supported. diff --git a/dev-docs/howto/vpn/helm/files/strongswan/charon-logging.conf b/dev-docs/howto/vpn/helm/files/strongswan/charon-logging.conf deleted file mode 100644 index 7114f6ad1..000000000 --- a/dev-docs/howto/vpn/helm/files/strongswan/charon-logging.conf +++ /dev/null @@ -1,11 +0,0 @@ -charon { - filelog { - stderr { - time_format = %b %e %T - ike_name = yes - default = 1 - ike = 2 - flush_line = yes - } - } -} diff --git a/dev-docs/howto/vpn/helm/files/tproxy-setup.sh b/dev-docs/howto/vpn/helm/files/tproxy-setup.sh deleted file mode 100644 index adbfc272d..000000000 --- a/dev-docs/howto/vpn/helm/files/tproxy-setup.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh - -set -eu - -### Pod IPs ### - -# Pod IPs are just NATed. - -iptables -t nat -N VPN_POST || iptables -t nat -F VPN_POST - -for cidr in ${VPN_PEER_CIDRS}; do - iptables -t nat -A VPN_POST -s "${cidr}" -d "${VPN_POD_CIDR}" -j MASQUERADE -done - -iptables -t nat -C POSTROUTING -j VPN_POST || iptables -t nat -A POSTROUTING -j VPN_POST - -### Service IPs ### - -# Service IPs need to be connected to locally to trigger the cgroup connect hook, thus we send them to the transparent proxy. - -# Packets with mark 1 are for tproxy and need to be delivered locally. -# For more information see: https://www.kernel.org/doc/Documentation/networking/tproxy.txt -pref=42 -table=42 -mark=0x1/0x1 -ip rule add pref "${pref}" fwmark "${mark}" lookup "${table}" -ip route replace local 0.0.0.0/0 dev lo table "${table}" - -iptables -t mangle -N VPN_PRE || iptables -t mangle -F VPN_PRE - -for cidr in ${VPN_PEER_CIDRS}; do - for proto in tcp udp; do - iptables -t mangle -A VPN_PRE -p "${proto}" -s "${cidr}" -d "${VPN_SERVICE_CIDR}" \ - -j TPROXY --tproxy-mark "${mark}" --on-port 61001 - done -done - -iptables -t mangle -C PREROUTING -j VPN_PRE || iptables -t mangle -A PREROUTING -j VPN_PRE diff --git a/dev-docs/howto/vpn/helm/files/wireguard-setup.sh b/dev-docs/howto/vpn/helm/files/wireguard-setup.sh deleted file mode 100644 index f97696a93..000000000 --- a/dev-docs/howto/vpn/helm/files/wireguard-setup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -set -eu - -dev=vpn_wg0 - -ip link add dev "${dev}" type wireguard -wg setconf "${dev}" /etc/wireguard/wg.conf -ip link set dev "${dev}" up - -for cidr in ${VPN_PEER_CIDRS}; do - ip route replace "${cidr}" dev "${dev}" -done diff --git a/dev-docs/howto/vpn/helm/templates/_helpers.tpl b/dev-docs/howto/vpn/helm/templates/_helpers.tpl index e090b562d..ab8a91e66 100644 --- a/dev-docs/howto/vpn/helm/templates/_helpers.tpl +++ b/dev-docs/howto/vpn/helm/templates/_helpers.tpl @@ -37,4 +37,8 @@ app.kubernetes.io/instance: {{ .Release.Name }} value: {{ .Values.podCIDR | quote }} - name: VPN_SERVICE_CIDR value: {{ .Values.serviceCIDR | quote }} +- name: VPN_FRONTEND_POD + value: {{ include "..fullname" . }}-frontend-0 +- name: VPN_MTU + value: {{ .Values.mtu | quote }} {{- end }} diff --git a/dev-docs/howto/vpn/helm/templates/configmaps.yaml b/dev-docs/howto/vpn/helm/templates/configmaps.yaml index 7f20c6760..d693a2835 100644 --- a/dev-docs/howto/vpn/helm/templates/configmaps.yaml +++ b/dev-docs/howto/vpn/helm/templates/configmaps.yaml @@ -1,22 +1,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "..fullname" . }}-tproxy + name: {{ include "..fullname" . }}-operator labels: {{- include "..labels" . | nindent 4 }} data: -{{ (.Files.Glob "files/tproxy-setup.sh").AsConfig | indent 2 }} +{{ (.Files.Glob "files/operator/*").AsConfig | indent 2 }} --- -{{- if .Values.wireguard.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "..fullname" . }}-wg - labels: {{- include "..labels" . | nindent 4 }} -data: -{{ (.Files.Glob "files/wireguard-setup.sh").AsConfig | indent 2 }} -{{- end }} ---- -{{ if .Values.ipsec.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -24,4 +13,3 @@ metadata: labels: {{- include "..labels" . | nindent 4 }} data: {{ (.Files.Glob "files/strongswan/*").AsConfig | indent 2 }} -{{- end }} diff --git a/dev-docs/howto/vpn/helm/templates/operator-deployment.yaml b/dev-docs/howto/vpn/helm/templates/operator-deployment.yaml new file mode 100644 index 000000000..1d82082cd --- /dev/null +++ b/dev-docs/howto/vpn/helm/templates/operator-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "..fullname" . }}-operator + labels: {{- include "..labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "..selectorLabels" . | nindent 6 }} + component: operator + template: + metadata: + labels: + {{- include "..selectorLabels" . | nindent 8 }} + component: operator + spec: + serviceAccountName: {{ include "..fullname" . }} + automountServiceAccountToken: true + containers: + - name: operator + image: {{ .Values.image | quote }} + command: ["/bin/operator.sh"] + env: {{- include "..commonEnv" . | nindent 10 }} diff --git a/dev-docs/howto/vpn/helm/templates/rbac.yaml b/dev-docs/howto/vpn/helm/templates/rbac.yaml new file mode 100644 index 000000000..83e68897c --- /dev/null +++ b/dev-docs/howto/vpn/helm/templates/rbac.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "..fullname" . }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "..fullname" . }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "patch"] +- apiGroups: ["cilium.io"] + resources: ["ciliumendpoints"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "..fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "..fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "..fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/dev-docs/howto/vpn/helm/templates/secrets.yaml b/dev-docs/howto/vpn/helm/templates/secrets.yaml index 9c050559b..7f9b6e870 100644 --- a/dev-docs/howto/vpn/helm/templates/secrets.yaml +++ b/dev-docs/howto/vpn/helm/templates/secrets.yaml @@ -1,15 +1,3 @@ -{{- if .Values.wireguard.enabled }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "..fullname" . }}-wg - labels: - {{- include "..labels" . | nindent 4 }} -data: - wg.conf: {{ include "wireguard.conf" . | b64enc }} -{{- end }} ---- -{{ if .Values.ipsec.enabled }} apiVersion: v1 kind: Secret metadata: @@ -18,4 +6,3 @@ metadata: {{- include "..labels" . | nindent 4 }} data: swanctl.conf: {{ include "strongswan.swanctl-conf" . | b64enc }} -{{- end }} diff --git a/dev-docs/howto/vpn/helm/templates/service.yaml b/dev-docs/howto/vpn/helm/templates/service.yaml index 6c4bc9755..65f9fe732 100644 --- a/dev-docs/howto/vpn/helm/templates/service.yaml +++ b/dev-docs/howto/vpn/helm/templates/service.yaml @@ -11,16 +11,9 @@ spec: component: frontend externalTrafficPolicy: Local ports: - {{- if .Values.ipsec.enabled }} - name: isakmp protocol: UDP port: 500 - name: ipsec-nat-t protocol: UDP port: 4500 - {{- end }} - {{- if .Values.wireguard.enabled }} - - name: wg - protocol: UDP - port: {{ .Values.wireguard.port }} - {{- end }} diff --git a/dev-docs/howto/vpn/helm/templates/strongswan-statefulset.yaml b/dev-docs/howto/vpn/helm/templates/strongswan-statefulset.yaml index f619373f1..314247ed0 100644 --- a/dev-docs/howto/vpn/helm/templates/strongswan-statefulset.yaml +++ b/dev-docs/howto/vpn/helm/templates/strongswan-statefulset.yaml @@ -1,4 +1,3 @@ -{{ if .Values.ipsec.enabled -}} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -15,64 +14,26 @@ spec: {{- include "..selectorLabels" . | nindent 8 }} component: frontend spec: - hostNetwork: false - initContainers: - - name: tproxy-setup - image: nixery.dev/busybox/iptables - command: ["/bin/sh", "-x", "/entrypoint.sh"] - env: {{- include "..commonEnv" . | nindent 10 }} - securityContext: - capabilities: - add: ["NET_ADMIN"] - volumeMounts: - - name: tproxy-setup - mountPath: "/entrypoint.sh" - subPath: "tproxy-setup.sh" - readOnly: true + hostPID: true containers: - - name: tproxy - # Image source: github.com/burgerdev/go-tproxy - image: ghcr.io/burgerdev/go-tproxy:latest - command: ["/tproxy", "--port=61001", "--nat=true"] - securityContext: - capabilities: - add: ["NET_RAW"] - name: strongswan - image: "nixery.dev/shell/strongswan" - command: ["/bin/sh", "-x", "/entrypoint.sh"] + image: {{ .Values.image | quote }} + command: ["/bin/strongswan.sh"] securityContext: capabilities: add: ["NET_ADMIN"] volumeMounts: - - name: strongswan - mountPath: "/entrypoint.sh" - subPath: "entrypoint.sh" - readOnly: true - - name: strongswan - mountPath: "/etc/strongswan.d/charon-logging.conf" - subPath: "charon-logging.conf" - readOnly: true - - name: strongswan + - name: config mountPath: "/etc/swanctl/swanctl.conf" subPath: "swanctl.conf" readOnly: true + - name: cilium-setup + image: {{ .Values.image | quote }} + command: ["/bin/sidecar.sh"] + env: {{- include "..commonEnv" . | nindent 10 }} + securityContext: + privileged: true volumes: - - name: tproxy-setup - configMap: - name: {{ include "..fullname" . }}-tproxy - - name: strongswan - projected: - sources: - - secret: - name: {{ include "..fullname" . }}-strongswan - items: - - key: swanctl.conf - path: swanctl.conf - - configMap: - name: {{ include "..fullname" . }}-strongswan - items: - - key: entrypoint.sh - path: entrypoint.sh - - key: charon-logging.conf - path: charon-logging.conf -{{- end }} + - name: config + secret: + secretName: {{ include "..fullname" . }}-strongswan diff --git a/dev-docs/howto/vpn/helm/templates/wireguard-secret.tpl b/dev-docs/howto/vpn/helm/templates/wireguard-secret.tpl deleted file mode 100644 index 99c23a7fe..000000000 --- a/dev-docs/howto/vpn/helm/templates/wireguard-secret.tpl +++ /dev/null @@ -1,14 +0,0 @@ -{{- define "wireguard.conf" }} -[Interface] -ListenPort = {{ .Values.wireguard.port }} -PrivateKey = {{ .Values.wireguard.private_key }} -[Peer] -PublicKey = {{ .Values.wireguard.peer_key }} -AllowedIPs = {{ join "," .Values.peerCIDRs }} -{{- if .Values.wireguard.endpoint }} -Endpoint = {{- .Values.wireguard.endpoint }} -{{- end }} -{{- if .Values.wireguard.keepAlive }} -PersistentKeepalive = {{- .Values.wireguard.keepAlive }} -{{- end }} -{{ end }} diff --git a/dev-docs/howto/vpn/helm/templates/wireguard-statefulset.yaml b/dev-docs/howto/vpn/helm/templates/wireguard-statefulset.yaml deleted file mode 100644 index f39b05cc7..000000000 --- a/dev-docs/howto/vpn/helm/templates/wireguard-statefulset.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{ if .Values.wireguard.enabled -}} -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "..fullname" . }}-frontend - labels: {{- include "..labels" . | nindent 4 }} -spec: - selector: - matchLabels: - {{- include "..selectorLabels" . | nindent 6 }} - component: frontend - template: - metadata: - labels: - {{- include "..selectorLabels" . | nindent 8 }} - component: frontend - spec: - hostNetwork: false - initContainers: - - name: tproxy-setup - image: nixery.dev/busybox/iptables - command: ["/bin/sh", "-x", "/entrypoint.sh"] - env: {{- include "..commonEnv" . | nindent 10 }} - securityContext: - capabilities: - add: ["NET_ADMIN"] - volumeMounts: - - name: tproxy-setup - mountPath: "/entrypoint.sh" - subPath: "tproxy-setup.sh" - readOnly: true - - name: wg-setup - image: "nixery.dev/busybox/wireguard-tools" - command: ["/bin/sh", "-x", "/etc/wireguard/wireguard-setup.sh"] - env: {{- include "..commonEnv" . | nindent 10 }} - securityContext: - capabilities: - add: ["NET_ADMIN"] - volumeMounts: - - name: wireguard - mountPath: "/etc/wireguard" - readOnly: true - containers: - - name: tproxy - # Image source: github.com/burgerdev/go-tproxy - image: ghcr.io/burgerdev/go-tproxy:latest - command: ["/tproxy", "--port=61001", "--nat=true"] - securityContext: - capabilities: - add: ["NET_RAW"] - volumes: - - name: tproxy-setup - configMap: - name: {{ include "..fullname" . }}-tproxy - - name: wireguard - projected: - sources: - - secret: - name: {{ include "..fullname" . }}-wg - items: - - key: wg.conf - path: wg.conf - - configMap: - name: {{ include "..fullname" . }}-wg - items: - - key: wireguard-setup.sh - path: wireguard-setup.sh -{{- end }} diff --git a/dev-docs/howto/vpn/helm/values.yaml b/dev-docs/howto/vpn/helm/values.yaml index 3d17833dc..607a2775d 100644 --- a/dev-docs/howto/vpn/helm/values.yaml +++ b/dev-docs/howto/vpn/helm/values.yaml @@ -8,32 +8,15 @@ serviceCIDR: "10.96.0.0/12" # on-prem IP ranges to expose to Constellation. Must contain at least one CIDR. peerCIDRs: [] - -# The sections below configure the VPN connectivity to the Constellation -# cluster. Exactly one `enabled` must be set to true. +# MTU to set on the VPN route. Leave empty if path MTU discovery is supported end-to-end. +# See also https://docs.strongswan.org/docs/5.9/howtos/forwarding.html#_mtumss_issues. +mtu: 1300 # IPSec configuration ipsec: - enabled: false # pre-shared key used for authentication psk: "" # Address of the peer's gateway router. peer: "" -# Wireguard configuration -wireguard: - enabled: false - - # If Wireguard is enabled, these fields for the Constellation side must be populated. - private_key: "" - peer_key: "" - - # Listening port of the Constellation Wireguard. - port: 51820 - - # Optional host:port of the on-prem Wireguard. - endpoint: "" - - # Optional interval for keep-alive packets in seconds. Setting this helps the on-prem server to - # discover a restarted Constellation VPN frontend. - keepAlive: "" +image: "ghcr.io/edgelesssys/constellation/vpn@sha256:88b6a0265052cb0a68d20d9b20e0d42ef15e7a80e5f71201ecf32e004de2356e" diff --git a/dev-docs/howto/vpn/on-prem-terraform/.terraform.lock.hcl b/dev-docs/howto/vpn/on-prem-terraform/.terraform.lock.hcl index add2cba63..a4e090c0d 100644 --- a/dev-docs/howto/vpn/on-prem-terraform/.terraform.lock.hcl +++ b/dev-docs/howto/vpn/on-prem-terraform/.terraform.lock.hcl @@ -2,50 +2,62 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.74.0" - constraints = "3.74.0" + version = "4.29.0" + constraints = "4.29.0" hashes = [ - "h1:4b15khHtc5OkIVEFg0W5QRwf/ov1WVQkXVdSiAcTCS8=", - "h1:ETVZfmulZQ435+lgFCkZRpfVOLyAxfDOwbPXFg3aLLQ=", - "h1:H3diAufZ5VDQKsQNYykVRaFTOUJ4gjFiT2VLYi574+w=", - "h1:LEdK8BxNSNiBQbtcJhQZKMMHDjmPpUsvDpr3Mzs93Tg=", - "h1:VfBB00BE0wvFiod7BlL+Cn6r2599MEi94hnAQ277ux8=", - "zh:0424c70152f949da1ec52ba96d20e5fd32fd22d9bd9203ce045d5f6aab3d20fc", - "zh:16dbf581d10f8e7937185bcdcceb4f91d08c919e452fb8da7580071288c8c397", - "zh:3019103bc2c3b4e185f5c65696c349697644c968f5c085af5505fed6d01c4241", - "zh:49bb56ebaed6653fdb913c2b2bb74fc8b5399e7258d1e89084f72c44ea1130dd", - "zh:85547666517f899d88620bd23a000a8f43c7dc93587c350eb1ea17bcb3e645c7", - "zh:8bed8b646ff1822d8764de68b56b71e5dd971a4b77eba80d47f400a530800bea", - "zh:8bfa6c70c004ba05ebce47f74f49ce872c28a68a18bb71b281a9681bcbbdbfa1", - "zh:a2ae9e38fda0695fb8aa810e4f1ce4b104bfda651a87923b307bb1728680d8b6", - "zh:beac1efe32f99072c892095f5ff46e40d6852b66679a03bc3acbe1b90fb1f653", - "zh:d8a6ca20e49ebe7ea5688d91233d571e2c2ccc3e41000c39a7d7031df209ea8e", + "h1:Bde/KCh2xGVCBx/JnixC3I2fmoRTwHXgsapfQ5QG8eg=", + "h1:IyINmgNiLfWx3Istkt5Mz+IJrDhSMhj3/qQeJlC4qS4=", + "h1:KEJAt0mJAACyIKUB5mCk/wqtxKMhivdeW8w6byz5Ll4=", + "h1:Y4gTSs+ZE5YSJVXG2qmsbXmv9Daq5aGM8Ip/GE6nev8=", + "h1:YtcHvTVfVBKbMCp9esoj527R1UK/hU0Zmo3pyQb8YhQ=", + "h1:atJdgnuqk+w3v4Zzhw2B1FZeYYA4su9JfanwNsx+c8o=", + "h1:c9tmtEdVTb9siGa3hVxPrMVl9ij5zijnD02JMHcHjrE=", + "h1:eN0KhMGVepEPpSA+YN5Kaz/v9PFKCafbkqqBzpLJf+g=", + "h1:hNVKlXTx2duXnR6SNKtyQMx7zSIlrxBu66Z0gbyfv3c=", + "h1:jC2GJo4VzTKnKociUDLVv8/+u9Mz+4scZrqbEasV+Y0=", + "h1:m3xYvc9X0pec0Zd1dpn82ALQ+6vwz56RnF/3CbkI2Eg=", + "zh:16590eea68c7c8aedb7af19f690eb34ab6636ef417b3fa9e06ca038fdb4c44b8", + "zh:1c907dfe44d00a54aa63d443004add90188f9a53ef3e919aff8aba92f715f42b", + "zh:258a0ff4198d80cae33c89091cd556d84c1b522c4416458484f23719a0cdf4cc", + "zh:587f5e9b2b33e51b18fb0f372025c961c3f57f2958b388459dd8432412650bda", + "zh:6318ca03bd9dbac272a75bb193688c7d4c4b45c7460289820528f31bcd6c3fe9", + "zh:63e4e8128e26e4c3e0c3b6582ef527245eb35eb5c80ad278dc675ebdf71edeaf", + "zh:845c898a27a84a15ba26e95ee66ac9563f81bc672b5ca216af82d87fe09bd5f8", + "zh:8fa6434fa212d5501185f0adc985d3a3c1e0f449c78f040a4ca640cb1e809cac", + "zh:9b49c0d72ab19aab43b2b48d23c5dddbbe29afae1569a987e6f20ed4c80ddf4c", + "zh:b14cc1ee5e3acf52490de7dd9791cce7953c0ee4bcccf0306aafd256568bd69f", + "zh:cd444836b2579fa42bfca2ae6145d394c41b6438b1ae01078c060bfaf803bb4d", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f937b5fdf49b072c0347408d0a1c5a5d822dae1a23252915930e5a82d1d8ce8b", ] } provider "registry.terraform.io/hashicorp/random" { - version = "3.6.0" - constraints = "3.6.0" + version = "3.7.2" + constraints = "3.7.2" hashes = [ - "h1:5KeoKSVKVHJW308uiTgslxFbjQAdWzBGUFK68vgMRWY=", - "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", - "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=", - "h1:p6WG1IPHnqx1fnJVKNjv733FBaArIugqy58HRZnpPCk=", - "h1:t0mRdJzegohRKhfdoQEJnv3JRISSezJRblN0HIe67vo=", - "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", - "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", - "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", - "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", - "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "h1:0hcNr59VEJbhZYwuDE/ysmyTS0evkfcLarlni+zATPM=", + "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=", + "h1:Def/iHM4HihJCIxQ8AYoxtoVL5lVlYx0V7bX91pxwgM=", + "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", + "h1:Lmv2TxyKKm9Vt4uxcPZHw1uf0Ax/yYizJlilbLSZN8E=", + "h1:hkKSY5xI4R1H4Yrg10HHbtOoxZif2dXa9HFPSbaVg5o=", + "h1:khu3pu9zeUMd6Ev+yH6cQ1S4+xpzx8wqwwFwADYGeRI=", + "h1:l35vnL76rzaOjhhJQiaWviW0noK2YzHeHN0/vIXJnHk=", + "h1:nWZjMYzp+nsqD3xslcihzq1Enxv33a7iC8/I8CTBcHA=", + "h1:pSMn/cwmyHB6V67lToGmCHfJFfzA711vV+E1cGP0LBg=", + "h1:w+NoF7vNMFS+qrU2XUEm0/wnuIZxPC733qOOfLVOdfk=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", - "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", - "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", - "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", - "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", - "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", ] } diff --git a/dev-docs/howto/vpn/on-prem-terraform/main.tf b/dev-docs/howto/vpn/on-prem-terraform/main.tf index f0a2b3088..b22ed0fd8 100644 --- a/dev-docs/howto/vpn/on-prem-terraform/main.tf +++ b/dev-docs/howto/vpn/on-prem-terraform/main.tf @@ -2,17 +2,21 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.74.0" + version = "4.29.0" } random = { source = "hashicorp/random" - version = "3.6.0" + version = "3.7.2" } } } provider "azurerm" { features {} + subscription_id = var.subscription_id + # This enables all resource providers. + # In the future, we might want to use `resource_providers_to_register` to registers just the ones we need. + resource_provider_registrations = "all" } locals { @@ -103,7 +107,7 @@ resource "azurerm_route_table" "route_table" { name = "vpn-routes" location = azurerm_resource_group.rg.location resource_group_name = azurerm_resource_group.rg.name - disable_bgp_route_propagation = false + bgp_route_propagation_enabled = false dynamic "route" { for_each = var.remote_ts diff --git a/dev-docs/howto/vpn/on-prem-terraform/variables.tf b/dev-docs/howto/vpn/on-prem-terraform/variables.tf index 652916678..86d74ea18 100644 --- a/dev-docs/howto/vpn/on-prem-terraform/variables.tf +++ b/dev-docs/howto/vpn/on-prem-terraform/variables.tf @@ -1,3 +1,9 @@ +variable "subscription_id" { + type = string + description = "Azure subscription ID. This can also be sourced from the ARM_SUBSCRIPTION_ID environment variable: https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs#subscription_id" + default = "" +} + variable "resource_group_location" { type = string default = "westeurope" diff --git a/dev-docs/miniconstellation/azure-terraform/.terraform.lock.hcl b/dev-docs/miniconstellation/azure-terraform/.terraform.lock.hcl index c110babaf..84170c34d 100644 --- a/dev-docs/miniconstellation/azure-terraform/.terraform.lock.hcl +++ b/dev-docs/miniconstellation/azure-terraform/.terraform.lock.hcl @@ -2,103 +2,121 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.74.0" - constraints = "3.74.0" + version = "4.29.0" + constraints = "4.29.0" hashes = [ - "h1:1kSiowd/tBNswp3iv7ePlzkP5llWihjHcY3pdXdJqVU=", - "h1:4b15khHtc5OkIVEFg0W5QRwf/ov1WVQkXVdSiAcTCS8=", - "h1:ETVZfmulZQ435+lgFCkZRpfVOLyAxfDOwbPXFg3aLLQ=", - "h1:H3diAufZ5VDQKsQNYykVRaFTOUJ4gjFiT2VLYi574+w=", - "h1:LEdK8BxNSNiBQbtcJhQZKMMHDjmPpUsvDpr3Mzs93Tg=", - "h1:OtJKZcMwrRNR84ylT1GgMwGR8KTxVOCkNifbjABlGj0=", - "h1:Rq+CNb+4u47dw20tlAeI2yxSOuDtLm+S/GZO2pneLyA=", - "h1:VfBB00BE0wvFiod7BlL+Cn6r2599MEi94hnAQ277ux8=", - "h1:YJ15rwD0G7lYc9OVh5GO4VTqcd2jhqegfgyqTJH1M/I=", - "h1:YvxxiqiwXjZdU53u3b9q49ezsIAb59KmdLLFkwkwFAs=", - "h1:xDRmcV40KrWttPYg/w0/IN/frS9K1twuyvqRNVZko44=", - "zh:0424c70152f949da1ec52ba96d20e5fd32fd22d9bd9203ce045d5f6aab3d20fc", - "zh:16dbf581d10f8e7937185bcdcceb4f91d08c919e452fb8da7580071288c8c397", - "zh:3019103bc2c3b4e185f5c65696c349697644c968f5c085af5505fed6d01c4241", - "zh:49bb56ebaed6653fdb913c2b2bb74fc8b5399e7258d1e89084f72c44ea1130dd", - "zh:85547666517f899d88620bd23a000a8f43c7dc93587c350eb1ea17bcb3e645c7", - "zh:8bed8b646ff1822d8764de68b56b71e5dd971a4b77eba80d47f400a530800bea", - "zh:8bfa6c70c004ba05ebce47f74f49ce872c28a68a18bb71b281a9681bcbbdbfa1", - "zh:a2ae9e38fda0695fb8aa810e4f1ce4b104bfda651a87923b307bb1728680d8b6", - "zh:beac1efe32f99072c892095f5ff46e40d6852b66679a03bc3acbe1b90fb1f653", - "zh:d8a6ca20e49ebe7ea5688d91233d571e2c2ccc3e41000c39a7d7031df209ea8e", + "h1:Bde/KCh2xGVCBx/JnixC3I2fmoRTwHXgsapfQ5QG8eg=", + "h1:IyINmgNiLfWx3Istkt5Mz+IJrDhSMhj3/qQeJlC4qS4=", + "h1:KEJAt0mJAACyIKUB5mCk/wqtxKMhivdeW8w6byz5Ll4=", + "h1:Y4gTSs+ZE5YSJVXG2qmsbXmv9Daq5aGM8Ip/GE6nev8=", + "h1:YtcHvTVfVBKbMCp9esoj527R1UK/hU0Zmo3pyQb8YhQ=", + "h1:atJdgnuqk+w3v4Zzhw2B1FZeYYA4su9JfanwNsx+c8o=", + "h1:c9tmtEdVTb9siGa3hVxPrMVl9ij5zijnD02JMHcHjrE=", + "h1:eN0KhMGVepEPpSA+YN5Kaz/v9PFKCafbkqqBzpLJf+g=", + "h1:hNVKlXTx2duXnR6SNKtyQMx7zSIlrxBu66Z0gbyfv3c=", + "h1:jC2GJo4VzTKnKociUDLVv8/+u9Mz+4scZrqbEasV+Y0=", + "h1:m3xYvc9X0pec0Zd1dpn82ALQ+6vwz56RnF/3CbkI2Eg=", + "zh:16590eea68c7c8aedb7af19f690eb34ab6636ef417b3fa9e06ca038fdb4c44b8", + "zh:1c907dfe44d00a54aa63d443004add90188f9a53ef3e919aff8aba92f715f42b", + "zh:258a0ff4198d80cae33c89091cd556d84c1b522c4416458484f23719a0cdf4cc", + "zh:587f5e9b2b33e51b18fb0f372025c961c3f57f2958b388459dd8432412650bda", + "zh:6318ca03bd9dbac272a75bb193688c7d4c4b45c7460289820528f31bcd6c3fe9", + "zh:63e4e8128e26e4c3e0c3b6582ef527245eb35eb5c80ad278dc675ebdf71edeaf", + "zh:845c898a27a84a15ba26e95ee66ac9563f81bc672b5ca216af82d87fe09bd5f8", + "zh:8fa6434fa212d5501185f0adc985d3a3c1e0f449c78f040a4ca640cb1e809cac", + "zh:9b49c0d72ab19aab43b2b48d23c5dddbbe29afae1569a987e6f20ed4c80ddf4c", + "zh:b14cc1ee5e3acf52490de7dd9791cce7953c0ee4bcccf0306aafd256568bd69f", + "zh:cd444836b2579fa42bfca2ae6145d394c41b6438b1ae01078c060bfaf803bb4d", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f937b5fdf49b072c0347408d0a1c5a5d822dae1a23252915930e5a82d1d8ce8b", ] } provider "registry.terraform.io/hashicorp/cloudinit" { - version = "2.3.2" - constraints = "2.3.2" + version = "2.3.7" + constraints = "2.3.7" hashes = [ - "h1:2jb+BfT5T96dXxUD2LQ6MtVHpXErd7ZybmMvdWE2jd4=", - "h1:Ar/DAbZQ9Nsj0BrqX6camrEE6U+Yq4E87DCNVqxqx8k=", - "h1:Vl0aixAYTV/bjathX7VArC5TVNkxBCsi3Vq7R4z1uvc=", - "h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=", - "h1:y+6FsU2STOpx6L6JOon4DVZoZPQgNoR2xR2WQ/EVxcQ=", - "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", - "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", - "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", - "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f", - "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422", - "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372", + "h1:/hny5kXmhcnuJDD1V+5XCrZOYDIqja2U47VM4DPEnBA=", + "h1:A9COAUjeBJ+fgYAI/PKtDs4Wzs50srFSY+KkfpSVGLw=", + "h1:Lt8lqrdNgZRlkOTwSXZTyuJkiVXnpwTsWAqHQPL6sIY=", + "h1:M9TpQxKAE/hyOwytdX9MUNZw30HoD/OXqYIug5fkqH8=", + "h1:coZHiZww6hWZoOoWw0p+6oeYb/tMh1uTvX1Y2ZzzXqE=", + "h1:dgBaiMxxU61piW30emM6251LMFW66TbKR+p5ylPZvqc=", + "h1:h1Pr6uNwq+iDEGrnQJEHzOTz+yVTW0AJgZrGXuoO4Qs=", + "h1:ht83gEvyri0BD3sata7BDhx31N/KbCECIozG7UM/kC8=", + "h1:iZ27qylcH/2bs685LJTKOKcQ+g7cF3VwN3kHMrzm4Ow=", + "h1:ll35IR++uaXwfwqZFFRWrvS0idO1mX43Y/embsaOe4k=", + "h1:rafNPmTutVTO2Horq45DG9Pjqrs+vx42oc7b/3aVGEc=", + "zh:06f1c54e919425c3139f8aeb8fcf9bceca7e560d48c9f0c1e3bb0a8ad9d9da1e", + "zh:0e1e4cf6fd98b019e764c28586a386dc136129fef50af8c7165a067e7e4a31d5", + "zh:1871f4337c7c57287d4d67396f633d224b8938708b772abfc664d1f80bd67edd", + "zh:2b9269d91b742a71b2248439d5e9824f0447e6d261bfb86a8a88528609b136d1", + "zh:3d8ae039af21426072c66d6a59a467d51f2d9189b8198616888c1b7fc42addc7", + "zh:3ef4e2db5bcf3e2d915921adced43929214e0946a6fb11793085d9a48995ae01", + "zh:42ae54381147437c83cbb8790cc68935d71b6357728a154109d3220b1beb4dc9", + "zh:4496b362605ae4cbc9ef7995d102351e2fe311897586ffc7a4a262ccca0c782a", + "zh:652a2401257a12706d32842f66dac05a735693abcb3e6517d6b5e2573729ba13", + "zh:7406c30806f5979eaed5f50c548eced2ea18ea121e01801d2f0d4d87a04f6a14", + "zh:7848429fd5a5bcf35f6fee8487df0fb64b09ec071330f3ff240c0343fe2a5224", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136", - "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7", - "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc", - "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040", - "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a", ] } provider "registry.terraform.io/hashicorp/random" { - version = "3.6.0" - constraints = "3.6.0" + version = "3.7.2" + constraints = "3.7.2" hashes = [ - "h1:5KeoKSVKVHJW308uiTgslxFbjQAdWzBGUFK68vgMRWY=", - "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", - "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=", - "h1:p6WG1IPHnqx1fnJVKNjv733FBaArIugqy58HRZnpPCk=", - "h1:t0mRdJzegohRKhfdoQEJnv3JRISSezJRblN0HIe67vo=", - "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", - "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", - "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", - "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", - "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "h1:0hcNr59VEJbhZYwuDE/ysmyTS0evkfcLarlni+zATPM=", + "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=", + "h1:Def/iHM4HihJCIxQ8AYoxtoVL5lVlYx0V7bX91pxwgM=", + "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", + "h1:Lmv2TxyKKm9Vt4uxcPZHw1uf0Ax/yYizJlilbLSZN8E=", + "h1:hkKSY5xI4R1H4Yrg10HHbtOoxZif2dXa9HFPSbaVg5o=", + "h1:khu3pu9zeUMd6Ev+yH6cQ1S4+xpzx8wqwwFwADYGeRI=", + "h1:l35vnL76rzaOjhhJQiaWviW0noK2YzHeHN0/vIXJnHk=", + "h1:nWZjMYzp+nsqD3xslcihzq1Enxv33a7iC8/I8CTBcHA=", + "h1:pSMn/cwmyHB6V67lToGmCHfJFfzA711vV+E1cGP0LBg=", + "h1:w+NoF7vNMFS+qrU2XUEm0/wnuIZxPC733qOOfLVOdfk=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", - "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", - "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", - "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", - "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", - "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", ] } provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.4" - constraints = "4.0.4" + version = "4.1.0" + constraints = "4.1.0" hashes = [ - "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=", - "h1:Wd3RqmQW60k2QWPN4sK5CtjGuO1d+CRNXgC+D4rKtXc=", - "h1:bNsvpX5EGuVxgGRXBQVLXlmq40PdoLp8Rfuh1ZmV7yY=", - "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", - "h1:rKKMyIEBZwR+8j6Tx3PwqBrStuH+J+pxcbCR5XN8WAw=", - "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", - "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", - "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", - "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", - "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", - "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", - "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", - "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", - "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", - "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", - "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "h1:4gd/jiOS0zJxjTd5Q4o/gOp24RxcuwQ/TxwjTYQNPz4=", + "h1:C0J7AsrVHVqnDT9tICDNaKvA9iH6WTLS2EYzCEegpx0=", + "h1:Ka8mEwRFXBabR33iN/WTIEW6RP0z13vFsDlwn11Pf2I=", + "h1:ReNkTkCM64bktu54eGwQc29rhIejMLQsYA6kYNyBWno=", + "h1:UklaKJOCynnEJbpCVN0zJKIJ3SvO7RQJ00/6grBatnw=", + "h1:ZHcr1WIomuU6ZV+dzEwAG1+52JP0e0d/+l7bo3N5p88=", + "h1:eZa3vbx1pbiwnajuKvGWE7jWK+nHQ8lcLc/mO6Rhf4o=", + "h1:iSgnCUoLGMkt31RlflnL09NyjpAH0DX6bb9QBw5IE9Y=", + "h1:uDtqTpFJOseNUlPDx4TT/lXf6ie3CarsimL7sYCiVH4=", + "h1:y9cHrgcuaZt592In6xQzz1lx7k/B9EeWrAb8K7QqOgU=", + "h1:zEv9tY1KR5vaLSyp2lkrucNJ+Vq3c+sTFK9GyQGLtFs=", + "zh:14c35d89307988c835a7f8e26f1b83ce771e5f9b41e407f86a644c0152089ac2", + "zh:2fb9fe7a8b5afdbd3e903acb6776ef1be3f2e587fb236a8c60f11a9fa165faa8", + "zh:35808142ef850c0c60dd93dc06b95c747720ed2c40c89031781165f0c2baa2fc", + "zh:35b5dc95bc75f0b3b9c5ce54d4d7600c1ebc96fbb8dfca174536e8bf103c8cdc", + "zh:38aa27c6a6c98f1712aa5cc30011884dc4b128b4073a4a27883374bfa3ec9fac", + "zh:51fb247e3a2e88f0047cb97bb9df7c228254a3b3021c5534e4563b4007e6f882", + "zh:62b981ce491e38d892ba6364d1d0cdaadcee37cc218590e07b310b1dfa34be2d", + "zh:bc8e47efc611924a79f947ce072a9ad698f311d4a60d0b4dfff6758c912b7298", + "zh:c149508bd131765d1bc085c75a870abb314ff5a6d7f5ac1035a8892d686b6297", + "zh:d38d40783503d278b63858978d40e07ac48123a2925e1a6b47e62179c046f87a", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb07f708e3316615f6d218cec198504984c0ce7000b9f1eebff7516e384f4b54", ] } diff --git a/dev-docs/miniconstellation/azure-terraform/main.tf b/dev-docs/miniconstellation/azure-terraform/main.tf index d0f1c8759..9aeeebe21 100644 --- a/dev-docs/miniconstellation/azure-terraform/main.tf +++ b/dev-docs/miniconstellation/azure-terraform/main.tf @@ -2,19 +2,19 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.74.0" + version = "4.29.0" } random = { source = "hashicorp/random" - version = "3.6.0" + version = "3.7.2" } tls = { source = "hashicorp/tls" - version = "4.0.4" + version = "4.1.0" } cloudinit = { source = "hashicorp/cloudinit" - version = "2.3.2" + version = "2.3.7" } } } @@ -22,6 +22,9 @@ terraform { provider "azurerm" { use_oidc = true features {} + # This enables all resource providers. + # In the future, we might want to use `resource_providers_to_register` to registers just the ones we need. + resource_provider_registrations = "all" } provider "tls" {} diff --git a/dev-docs/miniconstellation/setup-miniconstellation.sh b/dev-docs/miniconstellation/setup-miniconstellation.sh index fcc15d429..cbbd5d266 100755 --- a/dev-docs/miniconstellation/setup-miniconstellation.sh +++ b/dev-docs/miniconstellation/setup-miniconstellation.sh @@ -5,7 +5,6 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation # Start docker service and auto start on boot -# TODO(elchead) should be done in cloud-init but was not done in my test case sudo systemctl start docker.service && sudo systemctl enable docker.service echo "Waiting for docker service to be active..." # Wait at most 20min diff --git a/dev-docs/security-overview.md b/dev-docs/security-overview.md new file mode 100644 index 000000000..90393c182 --- /dev/null +++ b/dev-docs/security-overview.md @@ -0,0 +1,250 @@ +# Constellation - Security Protocols Overview + +The security of Constellation is based on a set of protocols. +The protocols are outlined in the following. +The following diagram sketches the basic trust relationships between the entities in a Constellation cluster. +![chain of trust](chain-of-trust.jpg) + +## Software components + +Abstractly, the Constellation software comprises three core components: +1. The command-line interface (CLI) executable, which is run by the cluster administrator on their local machine. +2. The node images, which are run inside Confidential VMs (CVMs). +Among other, each node image contains a Linux kernel and a user mode program called Bootstrapper. +3. The Constellation service containers, which are run on Kubernetes. +There are three core services: KeyService, JoinService, and VerificationService. + +## CLI: root of trust +The CLI executable is signed by Edgeless Systems. +To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [Sigstore project](https://www.sigstore.dev/). +There's a [step-by-step guide](https://docs.edgeless.systems/constellation/workflows/verify-cli) on how to verify CLI signatures based on Sigstore. + +The CLI contains the [measurements](https://github.com/edgelesssys/constellation/blob/edc0c7068ef4527efeaf584a2a35e0f51f58c426/internal/attestation/measurements/measurements_enterprise.go#L21) of the latest Constellation node image for all supported cloud platforms. +The CLI writes these measurements as part of the *attestation config* to a cluster's config file with the ["config generate" command](https://docs.edgeless.systems/constellation/workflows/config). +Note that Constellation currently uses 16 TPM-based runtime measurements for each cloud platform. +The purpose and source of the measurements are described in the [next section](#remote-attestation-of-nodes). +In addition to the measurements, the attestation config contains expected patch levels for the CPU microcode and the X.509 certificate of the CPU vendor's remote attestation infrastructure. +An example of an attestation config is given [below](#attestation-config). + +In case a different version of the node image is to be used, the corresponding measurements can be fetched using the CLI's ["config fetch-measurements" command](https://docs.edgeless.systems/constellation/reference/cli#constellation-config-fetch-measurements). +This command downloads the measurements and the corresponding signature from Edgeless Systems from https://cdn.confidential.cloud. +See for example the following files corresponding to node image v2.16.3: +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +In addition to the attestation config, the CLI contains the following data in hardcoded form: +* The [long-term public key of Edgeless Systems](https://github.com/edgelesssys/constellation/blob/edc0c7068ef4527efeaf584a2a35e0f51f58c426/internal/constants/constants.go#L264) to verify the signature of downloaded measurements. +* The [hashes of the expected Kubernetes binaries](https://github.com/edgelesssys/constellation/blob/edc0c7068ef4527efeaf584a2a35e0f51f58c426/internal/versions/versions.go#L199). +* The [Helm charts used for the installation of services](https://github.com/edgelesssys/constellation/tree/main/internal/constellation/helm/charts), which include hashes of the respective containers. +Note that the Helm charts and the hashes are [generated at build time](https://github.com/edgelesssys/constellation/blob/main/internal/constellation/helm/imageversion/imageversion.go). +A future version of the CLI will provide a command to print the Helm charts. + +## Cluster creation + +When a cluster is [created](https://docs.edgeless.systems/constellation/workflows/create), the CLI interacts with the API of the respective infrastructure provider, for example Azure, and launches CVMs with the applicable node image. +These CVMs are called *nodes*. +On each node, the Bootstrapper is launched. + +The CLI automatically selects one of the nodes as *first node*. +The CLI automatically verifies the first node's remote-attestation statement using the attestation config. +Details on remote attestation are given in the [next section](#remote-attestation-of-nodes). + +Based on the remote-attestation statement, the CLI and the Bootstrapper running on the first node set up a temporary TLS connection between them. +We refer to this type of connection as "attested TLS" (aTLS). +This connection is mainly used for three things (see the the [interface definition](https://github.com/edgelesssys/constellation/blob/main/bootstrapper/initproto/init.proto) for a comprehensive list of exchanged data): +1. The CLI sends the hashes of the expected Kubernetes binaries to the first node. +2. The CLI generates the [master secret](https://docs.edgeless.systems/constellation/architecture/keys#master-secret) of the to-be-created cluster and sends it to the first node. +3. The first node generates a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) and sends it to the CLI. +The kubeconfig file contains Kubernetes credentials for the CLI and the Kubernetes cluster's public key, among others. + +After this, the aTLS connection is closed and the Bootstrapper marks the node irrevocably as "initialized". +This mechanism prevents a node from accidentally or maliciously joining different clusters. +On all supported CVM platforms this is currently implemented by *extending* TPM register 15 with the unique ID of the cluster (`clusterID`). +More information can be found in the [Constellation documentation](https://docs.edgeless.systems/constellation/architecture/keys#cluster-identity). + +For [launch digest-based attestation](#remote-attestation-of-nodes) on future CVM platforms, an alternative would be to extend `clusterID` to the so called RTMR registers of Intel TDX. +TDX provides four RTMRs, which are automatically included in the `auxiliary data` part of a remote-attestation statement. +For AMD SEV-SNP, a different solution exists. + +## Remote attestation of nodes + +To identify themselves, nodes use the remote-attestation functionality of the underlying CVM platform. +Constellation supports Intel TDX and AMD SEV-SNP based platforms. +Abstractly, Intel TDX and AMD SEV-SNP hash the initial memory contents of the CVMs. +This hash is also called `launch digest`. +The launch digest is reflected in each remote-attestation statement that is requested by the software inside the CVM. +Abstractly, a remote-attestation statement `R` from a CVM looks as follows: + +``` +R = Sig-CPU(, , ) +``` + +The field `payload` is controlled by the software running inside the CVM. +In the case of a Constellation node, the `payload` is always the public key of the respective Bootstrapper running inside the CVM. +Thus, `R` can be seen as a certificate for that public key issued by the CPU. +Based on this, nodes establish attested TLS (aTLS) connections. +aTLS is used during [cluster creation](#cluster-creation) and when [growing a cluster](#cluster-growth). + +The field `auxiliary data` is populated automatically by the CVM platform and, among others, includes information like CPU firmware versions. + +Note that this description of `R` is highly abstract. + +### Measurements + +In the ideal case, the underlying CVM platform does not inject any of its own software into a CVM. +In that case, a Constellation node image can contain its own firmware/UEFI. +This allows for the creation of node images for which the launch digest covers all defining parts of a node, including the firmware, the kernel, the kernel command line, and the disk image. +In this case, the launch digest is the only measurement that's required to verify the identity and integrity of a node. + +### Measured Boot as fallback + +However, currently, all supported CVM platforms (AWS, Azure, and GCP) inject custom firmware into CVMs. +Thus, in practice, Constellation relies on conventional [measured boot](https://docs.edgeless.systems/constellation/architecture/images#measured-boot) to reflect the identity and integrity of nodes. + +In measured boot, in general, the software components involved in the boot process of a system are "measured" into the 16 platform configuration registers (PCRs) of a Trusted Platform Module (TPM). +The values of these registers are also called "runtime measurements". +All supported CVM platforms provide TPMs to CVMs. + +With measured boot, Constellation relies on TPM-based remote attestation for nodes. +TPM-based remote attestation is similar to confidential computing-based remote attestation. Instead of the value `R`, the value `R'` is used. + +``` +R' = Sig-TPM(, ) +``` + +The field `auxiliary data` is populated automatically by the TPM and most notably contains the 16 PCRs. +Constellation uses the field `payload` as usual and sets it to the public key of the respective CVM's Bootstrapper. +When verifying `R'`, Constellation compares the 16 PCRs to those given in the attestation config. + +#### Differences between CVM platforms + +Each supported CVM platform populates the 16 PCRs in different ways. Details can be found in the [Constellation documentation](https://docs.edgeless.systems/constellation/architecture/attestation#runtime-measurements). +Sig-TPM itself is also verified differently for each cloud. + +Currently, on AWS and GCP the TPM implementation resides outside the CVM. +On Azure, the TPM implementation is part of the injected firmware and resides inside the CVM. +More information can be found in the [Constellation documentation](https://docs.edgeless.systems/constellation/overview/clouds). + +## Kubernetes bootstrapping on the first node + +The Bootstrapper on the first node downloads and verifies the Kubernetes binaries, using the hashes it received from the CLI. +These binaries include for example [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/), [kube-apiserver](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/), and [etcd](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/). +With these, the Bootstrapper creates a single-node Kubernetes cluster. +Etcd is a distributed key-value store that Kubernetes uses to store configuration data for services. +The etcd agent runs on each control-plane node of a cluster. +The agents use mTLS for communication between them. +Etcd uses the Raft protocol (over mTLS) to distribute state between nodes. +All essential configuration data of a cluster is kept in etcd. + +Initially, the Bootstrapper on the first node [writes](https://github.com/edgelesssys/constellation/blob/d65987cb15cf9ebdbbd2975f177937c1acbc90f8/bootstrapper/internal/kubernetes/kubernetes.go#L173) the hashes of the expected Kubernetes binaries to a specific key in etcd. + +Next, the CLI the connects to the Kubernetes API server (kube-apiserver) using the kubeconfig file it received from the Bootstrapper. +This results in an mTLS connection between the CLI and the Kubernetes API server. +The CLI uses this connection for two essential operations at the Kubernetes level: + +1. It writes the attestation config to a specific key in etcd. +1. It executes the [hardcoded Helm charts](#cli-root-of-trust), which, most notably, install the three core services KeyService, JoinService, and VerificationService, the [constellation-node-operator](https://github.com/edgelesssys/constellation/tree/main/operators/constellation-node-operator), and a small number of standard services like Cilium and cert-manager. + +The latter causes the first node to download, verify, and run the containers defined in the Helm charts. +The containers that are specific to Constellation are hosted at `ghcr.io/edgelesssys`. + +After this, the Constellation cluster is operational on the first node. + +## Cluster growth + +Additional nodes can now join the cluster - either as control-plane nodes or as worker nodes. +For both, the process for joining the cluster is identical. +First, the Bootstrapper running on a *new node* contacts the JoinService of the existing cluster. +The JoinService verifies the remote-attestation statement of the new node using the attestation config stored in etcd. +On success, an aTLS connection between the two is created, which is used mainly for the following (see the the [interface definition](https://github.com/edgelesssys/constellation/blob/main/joinservice/joinproto/join.proto) for a comprehensive list of exchanged data): + +1. The new node sends a certificate signing request (CSR) for its *node certificate* to the JoinService. +1. The JoinService issues a corresponding certificate with a lifetime of one year and sends it to the new node. +The JoinService uses the signing key of the Kubernetes cluster for this, which is [generated by kubeadm](https://kubernetes.io/docs/setup/best-practices/certificates/). Note that the lifetime of the node certificate is a best practice only, as Constellation relies on the untrusted infrastructure to provide time when validating certificates. +1. The JoinService sends a [kubeadm token](https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-token/) to the new node. +1. The JoinService sends the hashes of the expected Kubernetes binaries to the new node. +1. The JoinService sends encryption key for the new node's local storage. +This key is generated and managed by the cluster's KeyService. +1. The JoinService sends the certificate of the cluster's control plane to the new node. + +After this, the aTLS connection is closed and the node is marked as "initialized" in the same way as described [above](#cluster-creation). + +The Bootstrapper downloads, verifies, and runs the given Kubernetes binaries. +Further, the Bootstrapper uses the kubeadm token to download the configuration of the cluster from the Kubernetes API server. +The kubeadm token is never used after this. + +The kubelet on the new node uses its own node certificate and the certificate of the cluster's control plane (which the new node both received from the JoinService) to establish an mTLS connection with the cluster's Kubernetes API server. +Once connected, the new node registers itself as control-plane node or worker node of the cluster. +This process uses the standard Kubernetes mechanisms for adding nodes. + +In Constellation, a virtual private network (VPN) exists between all nodes of a cluster. +This VPN is created with the help of Cilium. +To join this VPN, the new node generates WireGuard credentials for itself and writes the public key to etcd via the mTLS connection with the Kubernetes API server. +It also downloads the public keys of existing nodes from etcd. +Subsequently, the Cilium agents running on other nodes fetch the new node's public key from etcd as well. + +Note that etcd communication between nodes is an exception: This traffic always goes via mTLS based on node certificates. + +## Cluster upgrade + +Whenever a cluster is [upgraded](https://docs.edgeless.systems/constellation/workflows/upgrade), the CLI connects to the Kubernetes API server and, essentially, updates the following data in etcd: + +1. The attestation config +1. The hashes of the expected Kubernetes binaries + +Further, the CLI applies updated Helm charts to update the cluster's services. +Again, these Helm charts are hardcoded in the CLI. +See the [implementation](https://github.com/edgelesssys/constellation/blob/d65987cb15cf9ebdbbd2975f177937c1acbc90f8/cli/internal/cmd/apply.go#L358) of the `apply()` function for a sequence diagram of all steps. +Subsequently, the constellation-node-operator replaces existing nodes with new ones. +New nodes go through the [usual process for joining the cluster](#cluster-growth). + +## Examples + +This section gives real life examples of key data structures and the corresponding commands to retrieve those. + +### Attestation config + +```bash +kubectl -n kube-system get cm join-config -o json +``` +```json +{ + "apiVersion": "v1", + "binaryData": { + "measurementSalt": "2A4Fzfdr/61XbJvk1PDqzh0R4rVnEujyXudsfgRZzUY=" + }, + "data": { + "attestationConfig": "{\"measurements\":{\"1\":{\"expected\":\"3695dcc55e3aa34027c27793c85c723c697d708c42d1f73bd6fa4f26608a5b24\",\"warnOnly\":true},\"11\":{\"expected\":\"f09cef0d077127fb26bc8d013fc09e13afbb70f0f734ced98f46666544998efe\",\"warnOnly\":true},\"12\":{\"expected\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"warnOnly\":true},\"13\":{\"expected\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"warnOnly\":true},\"14\":{\"expected\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"warnOnly\":true},\"15\":{\"expected\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"warnOnly\":true},\"2\":{\"expected\":\"3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969\",\"warnOnly\":true},\"3\":{\"expected\":\"3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969\",\"warnOnly\":true},\"4\":{\"expected\":\"e5020193148fbad0dbaf618fb3ef15665c72ff87a54e24b2d8f5bdf9719bb50b\",\"warnOnly\":true},\"6\":{\"expected\":\"3d458cfe55cc03ea1f443f1562beec8df51c75e14a9fcf9a7234a13f198e7969\",\"warnOnly\":true},\"8\":{\"expected\":\"0000000000000000000000000000000000000000000000000000000000000000\",\"warnOnly\":true},\"9\":{\"expected\":\"37ef16fd0ae8d2fb3b1914f0b8ff046e765b57fec6739d2ebf1fd4d182071437\",\"warnOnly\":true}},\"bootloaderVersion\":\"latest\",\"teeVersion\":\"latest\",\"snpVersion\":\"latest\",\"microcodeVersion\":\"latest\",\"amdRootKey\":\"-----BEGIN CERTIFICATE-----\\nMIIGYzCCBBKgAwIBAgIDAQAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAIC\\nBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDAS\\nBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEg\\nQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZp\\nY2VzMRIwEAYDVQQDDAlBUkstTWlsYW4wHhcNMjAxMDIyMTcyMzA1WhcNNDUxMDIy\\nMTcyMzA1WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDAS\\nBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5j\\nZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLU1pbGFuMIICIjANBgkqhkiG\\n9w0BAQEFAAOCAg8AMIICCgKCAgEA0Ld52RJOdeiJlqK2JdsVmD7FktuotWwX1fNg\\nW41XY9Xz1HEhSUmhLz9Cu9DHRlvgJSNxbeYYsnJfvyjx1MfU0V5tkKiU1EesNFta\\n1kTA0szNisdYc9isqk7mXT5+KfGRbfc4V/9zRIcE8jlHN61S1ju8X93+6dxDUrG2\\nSzxqJ4BhqyYmUDruPXJSX4vUc01P7j98MpqOS95rORdGHeI52Naz5m2B+O+vjsC0\\n60d37jY9LFeuOP4Meri8qgfi2S5kKqg/aF6aPtuAZQVR7u3KFYXP59XmJgtcog05\\ngmI0T/OitLhuzVvpZcLph0odh/1IPXqx3+MnjD97A7fXpqGd/y8KxX7jksTEzAOg\\nbKAeam3lm+3yKIcTYMlsRMXPcjNbIvmsBykD//xSniusuHBkgnlENEWx1UcbQQrs\\n+gVDkuVPhsnzIRNgYvM48Y+7LGiJYnrmE8xcrexekBxrva2V9TJQqnN3Q53kt5vi\\nQi3+gCfmkwC0F0tirIZbLkXPrPwzZ0M9eNxhIySb2npJfgnqz55I0u33wh4r0ZNQ\\neTGfw03MBUtyuzGesGkcw+loqMaq1qR4tjGbPYxCvpCq7+OgpCCoMNit2uLo9M18\\nfHz10lOMT8nWAUvRZFzteXCm+7PHdYPlmQwUw3LvenJ/ILXoQPHfbkH0CyPfhl1j\\nWhJFZasCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSFrBrRQ/fI\\nrFXUxR1BSKvVeErUUzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuG\\nKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvTWlsYW4vY3JsMEYGCSqG\\nSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZI\\nAWUDBAICBQCiAwIBMKMDAgEBA4ICAQC6m0kDp6zv4Ojfgy+zleehsx6ol0ocgVel\\nETobpx+EuCsqVFRPK1jZ1sp/lyd9+0fQ0r66n7kagRk4Ca39g66WGTJMeJdqYriw\\nSTjjDCKVPSesWXYPVAyDhmP5n2v+BYipZWhpvqpaiO+EGK5IBP+578QeW/sSokrK\\ndHaLAxG2LhZxj9aF73fqC7OAJZ5aPonw4RE299FVarh1Tx2eT3wSgkDgutCTB1Yq\\nzT5DuwvAe+co2CIVIzMDamYuSFjPN0BCgojl7V+bTou7dMsqIu/TW/rPCX9/EUcp\\nKGKqPQ3P+N9r1hjEFY1plBg93t53OOo49GNI+V1zvXPLI6xIFVsh+mto2RtgEX/e\\npmMKTNN6psW88qg7c1hTWtN6MbRuQ0vm+O+/2tKBF2h8THb94OvvHHoFDpbCELlq\\nHnIYhxy0YKXGyaW1NjfULxrrmxVW4wcn5E8GddmvNa6yYm8scJagEi13mhGu4Jqh\\n3QU3sf8iUSUr09xQDwHtOQUVIqx4maBZPBtSMf+qUDtjXSSq8lfWcd8bLr9mdsUn\\nJZJ0+tuPMKmBnSH860llKk+VpVQsgqbzDIvOLvD6W1Umq25boxCYJ+TuBoa4s+HH\\nCViAvgT9kf/rBq1d+ivj6skkHxuzcxbk1xv6ZGxrteJxVH7KlX7YRdZ6eARKwLe4\\nAFZEAwoKCQ==\\n-----END CERTIFICATE-----\\n\",\"amdSigningKey\":\"\"}" + }, + "kind": "ConfigMap", + "metadata": { + "creationTimestamp": "2024-09-25T11:11:50Z", + "name": "join-config", + "namespace": "kube-system", + "resourceVersion": "387", + "uid": "fdd0d5eb-cf58-4608-99c9-eede08895615" + } +} +``` +### Hashes of Kubernetes binaries +```bash +kubectl -n kube-system get cm k8s-components-sha256-7b73c7675df78e5753b6e0fc86a9982127fd16141837599d5ce16df6bfe6a2a0 -o json +``` +```json +{ + "apiVersion": "v1", + "data": { + "cluster-version": "v1.29.8", + "components": "[{\"url\":\"https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz\",\"hash\":\"sha256:c2485ddb3ffc176578ae30ae58137f0b88e50f7c7f2af7d53a569276b2949a33\",\"install_path\":\"/opt/cni/bin\",\"extract\":true},{\"url\":\"https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz\",\"hash\":\"sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb\",\"install_path\":\"/run/state/bin\",\"extract\":true},{\"url\":\"https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubelet\",\"hash\":\"sha256:df6e130928403af8b4f49f1197e26f2873a147cd0e23aa6597a26c982c652ae0\",\"install_path\":\"/run/state/bin/kubelet\"},{\"url\":\"https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubeadm\",\"hash\":\"sha256:fe054355e0ae8dc35d868a3d3bc408ccdff0969c20bf7a231ae9b71484e41be3\",\"install_path\":\"/run/state/bin/kubeadm\"},{\"url\":\"https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubectl\",\"hash\":\"sha256:038454e0d79748aab41668f44ca6e4ac8affd1895a94f592b9739a0ae2a5f06a\",\"install_path\":\"/run/state/bin/kubectl\"},{\"url\":\"data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjhAc2hhMjU2OjZmNzJmYTkyNmM5YjA1ZTEwNjI5ZmUxYTA5MmZkMjhkY2Q2NWI0ZmRmZDBjYzdiZDU1Zjg1YTU3YTZiYTFmYTUifV0=\",\"install_path\":\"/opt/kubernetes/patches/kube-apiserver+json.json\"},{\"url\":\"data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjhAc2hhMjU2OjZmMjdkNjNkZWQyMDYxNGM2ODU1NGI0NzdjZDdhNzhlZGE3OGE0OThhOTJiZmU4OTM1Y2Y5NjRjYTViNzRkMGIifV0=\",\"install_path\":\"/opt/kubernetes/patches/kube-controller-manager+json.json\"},{\"url\":\"data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjhAc2hhMjU2OmRhNzRhNjY2NzVkOTVlMzllYzI1ZGE1ZTcwNzI5ZGE3NDZkMGZhMGIxNWVlMGRhODcyYWM5ODA1MTliYzI4YmQifV0=\",\"install_path\":\"/opt/kubernetes/patches/kube-scheduler+json.json\"},{\"url\":\"data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2V0Y2Q6My41LjEyLTBAc2hhMjU2OjQ0YThlMjRkY2JiYTM0NzBlZTFmZWUyMWQ1ZTg4ZDEyOGM5MzZlOWI1NWQ0YmM1MWZiZWY4MDg2ZjhlZDEyM2IifV0=\",\"install_path\":\"/opt/kubernetes/patches/etcd+json.json\"}]" + }, + "immutable": true, + "kind": "ConfigMap", + "metadata": { + "creationTimestamp": "2024-09-25T11:11:50Z", + "name": "k8s-components-sha256-7b73c7675df78e5753b6e0fc86a9982127fd16141837599d5ce16df6bfe6a2a0", + "namespace": "kube-system", + "resourceVersion": "356", + "uid": "6389c186-3bc8-4470-8af5-f6fed1addd69" + } +} +``` \ No newline at end of file diff --git a/dev-docs/workflows/attestationconfigapi.md b/dev-docs/workflows/attestationconfigapi.md index 5da8eda35..4881497ce 100644 --- a/dev-docs/workflows/attestationconfigapi.md +++ b/dev-docs/workflows/attestationconfigapi.md @@ -8,10 +8,10 @@ This estimate might make manual intervention necessary when a global rollout did ### Manually delete a version ``` -COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY="$(cat $PATH_TO_KEY)" AWS_ACCESS_KEY_ID=$ID AWS_ACCESS_KEY=$KEY bazel run //internal/api/attestationconfigapi/cli delete -- --version 2023-09-02-12-52 +COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY="$(cat $PATH_TO_KEY)" AWS_ACCESS_KEY_ID=$ID AWS_ACCESS_KEY=$KEY bazel run //internal/api/attestationconfigapi/cli -- delete azure-sev-snp attestation-report 2025-01-18-09-15 ``` ### Manually upload a version ``` -COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY="$(cat $PATH_TO_KEY)" AWS_ACCESS_KEY_ID=$ID AWS_ACCESS_KEY=$KEY bazel run //internal/api/attestationconfigapi/cli -- --force --version 2023-09-02-12-52 --maa-claims-path "${path}" +COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY="$(cat $PATH_TO_KEY)" AWS_ACCESS_KEY_ID=$ID AWS_ACCESS_KEY=$KEY bazel run //internal/api/attestationconfigapi/cli -- upload azure-sev-snp attestation-report 2025-01-18-09-15 --force ``` diff --git a/dev-docs/workflows/bazel.md b/dev-docs/workflows/bazel.md index 71fda6bc5..8dc38b7ea 100644 --- a/dev-docs/workflows/bazel.md +++ b/dev-docs/workflows/bazel.md @@ -3,6 +3,13 @@ Bazel is the primary build system for this project. It is used to build all Go code and will be used to build all artifacts in the future. Still, we aim to keep the codebase compatible with `go build` and `go test` as well. Whenever Go code is changed, you will have to run `bazel run //:tidy` to regenerate the Bazel build files for Go code. +Additionally, you need to update `MODULE.bazel`, together with `MODULE.bazel.lock`: + +``` +# if the steps below fail, try to recreate the lockfile from scratch by deleting it +bazel mod deps --lockfile_mode=update +bazel mod tidy +``` ## Bazel commands @@ -46,17 +53,6 @@ Also note that some errors shown in `check` (non-silent mode) by `golicenses_che * `bazel query //subfolder` - list all targets in a subfolder * `bazel cquery --output=files //subfolder:target` - get location of a build artifact -### (Optional) Remote caching and execution - -We use BuildBuddy for remote caching (and maybe remote execution in the future). To use it, you need to join the BuildBuddy organization and get an API key. Then, you can write it to `~/.bazelrc`: - -```sh -build --remote_header=x-buildbuddy-api-key= -``` - -To use the remote cache, build the project with `bazel build --config remote_cache //path/to:target`. -You can also copy the `remote_cache` config from `.bazelrc` to your `~/.bazelrc` and remove the `remote_cache` prefix to make it the default. - ## Setup ### VS Code integration diff --git a/dev-docs/workflows/build-develop-deploy.md b/dev-docs/workflows/build-develop-deploy.md index cc43f6b1a..4f596ddcb 100644 --- a/dev-docs/workflows/build-develop-deploy.md +++ b/dev-docs/workflows/build-develop-deploy.md @@ -15,13 +15,22 @@ Prerequisites: ### Linux -* If you don't want to perform any setup, you can get a shell with Bazel and all required dependencies by running: +If you don't want to perform any setup, you can get a shell with Bazel and all required dependencies by running: - ```sh - # better would be: nix develop -i - # but this doesn't play nice with bashrc, colored output and non-hermetic tools - nix develop - ``` +```sh +# better would be: nix develop -i +# but this doesn't play nice with bashrc, colored output and non-hermetic tools +nix develop +``` + +Or activate [direnv](https://direnv.net/) to automatically enter the nix shell. +It is recommended to use [nix-direnv](https://github.com/nix-community/nix-direnv). +If your system ships outdated bash, [install direnv](https://direnv.net/docs/installation.html) via package manager. +Additionally, you may want to add the [vscode extension](https://github.com/direnv/direnv-vscode). + +```sh +direnv allow +``` ### Mac diff --git a/dev-docs/workflows/bump-go-version.md b/dev-docs/workflows/bump-go-version.md index a8e897308..f2736179b 100644 --- a/dev-docs/workflows/bump-go-version.md +++ b/dev-docs/workflows/bump-go-version.md @@ -1,17 +1,41 @@ # Bump Go version + `govulncheck` from the bazel `check` target will fail if our code is vulnerable, which is often the case when a patch version was released with security fixes. ## Steps -1. Replace "1.xx.x" with the new version (see [example](https://github.com/edgelesssys/constellation/commit/9e1a0c06bfda0171958f0776633a9a53f521144d)) -2. Update the nix hash +Replace `"1.xx.x"` with the new version in [MODULE.bazel](/MODULE.bazel): - Once updated run `bazel run //:tidy` and you will see a failure such as: +```starlark +go_sdk = use_extension("@io_bazel_rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download( + name = "go_sdk", + patches = ["//3rdparty/bazel/org_golang:go_tls_max_handshake_size.patch"], + version = "1.xx.x", <--- Replace this one + ~~~~~~~~ +) - ``` - > error: hash mismatch in fixed-output derivation '/nix/store/r85bdj6vrim7m5vlybdmzgca7d0kcb4n-go1.21.4.src.tar.gz.drv': - > specified: sha256-GG8rb4yLcE5paCGwmrIEGlwe4T3LwxVqE63PdZMe5Ig= - > got: sha256-R7Jqg9K2WjwcG8rOJztpvuSaentRaKdgTe09JqN714c= - ``` - Simple replace the hash with the got value. -3. Ask @katexochen to build the thing and push it into his cache. +``` + +Replace `go-version: "1.xx.x"` with the new version in all GitHub actions/workflows, our go.mod files and Containerfiles. +You can use the following command to find replace all instances of `go-version: "1.xx.x"` in the `.github` directory: + +```bash +OLD_VERSION="1.xx.x" +NEW_VERSION="1.xx.y" +find .github -type f -exec sed -i "s/go-version: \"${OLD_VERSION}\"/go-version: \"${NEW_VERSION}\"/g" {} \; +sed -i "s/go ${OLD_VERSION}/go ${NEW_VERSION}/g" go.mod +sed -i "s/go ${OLD_VERSION}/go ${NEW_VERSION}/g" hack/tools/go.mod +sed -i "s/${OLD_VERSION}/${NEW_VERSION}/g" go.work +sed -i "s/GO_VER=${OLD_VERSION}/GO_VER=${NEW_VERSION}/g" 3rdparty/gcp-guest-agent/Dockerfile +``` + +Or manually: + +```yaml +- name: Setup Go environment + uses: actions/setup-go@v5 + with: + go-version: "1.xx.x" <--- Replace this one + ~~~~~~~~ +``` diff --git a/dev-docs/workflows/github-actions.md b/dev-docs/workflows/github-actions.md index ef97ed332..89f9f5d3c 100644 --- a/dev-docs/workflows/github-actions.md +++ b/dev-docs/workflows/github-actions.md @@ -24,7 +24,7 @@ Here are some examples for test suites you might want to run. Values for `sonobu * `--mode certified-conformance` * For K8s conformance certification test suite -Check [Sonobuoy docs](https://sonobuoy.io/docs/latest/e2eplugin/) for more examples. +Check [Sonobuoy docs](https://sonobuoy.io/docs/v0.57.1/e2eplugin/) for more examples. When using `--mode` be aware that `--e2e-focus` and `e2e-skip` will be overwritten. [Check in the source code](https://github.com/vmware-tanzu/sonobuoy/blob/e709787426316423a4821927b1749d5bcc90cb8c/cmd/sonobuoy/app/modes.go#L130) what the different modes do. diff --git a/dev-docs/workflows/marketplace-images.md b/dev-docs/workflows/marketplace-images.md deleted file mode 100644 index d19e2b250..000000000 --- a/dev-docs/workflows/marketplace-images.md +++ /dev/null @@ -1,27 +0,0 @@ -# Using Marketplace Images in Constellation - -This document explains the steps a user needs to take to run Constellation with dynamic billing via the cloud marketplaces. - -## AWS - -Marketplace Images on AWS are not available yet. - -## Azure - -On Azure, to use a marketplace image, ensure that the subscription has accepted the agreement to use marketplace images: - -```bash -az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation -``` - -Then, set the VMs to use the marketplace image in the `constellation-conf.yaml` file: - -```bash -yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml -``` - -And ensure that the cluster uses a release image (i.e. `.image=vX.Y.Z` in the `constellation-conf.yaml` file). Afterwards, proceed with the cluster creation as usual. - -## GCP - -Marketplace Images on GCP are not available yet. diff --git a/dev-docs/workflows/marketplace-publishing.md b/dev-docs/workflows/marketplace-publishing.md new file mode 100644 index 000000000..aa0513b2f --- /dev/null +++ b/dev-docs/workflows/marketplace-publishing.md @@ -0,0 +1,33 @@ +# Publishing Marketplace Images + +Constellation release images need to be manually published to AWS and Azure marketplaces due to the lack of automation features. +On GCP, marketplace image publishing is automated and takes place on release. + +This document explains how to perform the uploading on AWS and Azure. + +## AWS + +1. Log in to the [AWS marketplace management portal](https://aws.amazon.com/marketplace/management/) with your regular developer AWS account. +2. Select "Products -> Server -> Constellation" in the top menu. +3. Select "Versions" in the main menu and press "Add version". +4. Fill in the form. + 1. Enter the semantic version of the release (i.e. `vX.Y.Z`) as "Version title". + 2. Set the version tag in "Release notes" to the same version. + 3. For the "Amazon Machine Image (AMI) ID", enter the AMI ID of the release (SEV-SNP) image. This can be found in the regular + [AWS console](https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Images:visibility=owned-by-me;search=:constellation-v;v=3;$case=tags:false%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:creationDate). + 4. For "IAM access role ARN", enter `arn:aws:iam::795746500882:role/constellation-marketplace-ingest`. +5. Leave the other fields as they are and press "Add version". +6. Wait for the [request](https://aws.amazon.com/marketplace/management/requests) to be processed and available before publishing the release. + +## Azure + +1. Log in to the [Microsoft partner center](https://partner.microsoft.com/en-us/dashboard/home) with your regular developer Microsoft account. +2. Select "Marketplace offers -> Constellation -> Constellation Node" in the main menu. +3. Select "Technical configuration" in the sidebar on the left. +4. Select "Add VM Image". + 1. For the "Version number", enter the semantic version of the release without the `v` prefix. If the release version is `vX.Y.Z`, enter `X.Y.Z`. + 2. Press "Add a gallery image" and select the corresponding "Constellation_CVM" image version in the menu. + 3. Press "Save VM image". +5. **IMPORTANT**: Hit **Save draft**. Do **NOT** hit "Review and publish" directly. +6. **After** saving the draft, hit "Review and publish". +7. Go back to the [offer's home page](https://partner.microsoft.com/en-us/dashboard/commercial-marketplace/offers/a53ac90b-06f7-4a20-a845-8607ca352e61/overview) and wait for the process to complete before publishing the release. diff --git a/dev-docs/workflows/qemu.md b/dev-docs/workflows/qemu.md index 3f294cc8b..dd327d31b 100644 --- a/dev-docs/workflows/qemu.md +++ b/dev-docs/workflows/qemu.md @@ -20,7 +20,7 @@ Follow the steps in our [libvirt readme](../../nix/container/README.md) if you w ### Install required packages -[General reference](https://ubuntu.com/server/docs/virtualization-libvirt) +[General reference](https://documentation.ubuntu.com/server/how-to/virtualisation/libvirt/) ```shell-session sudo apt install qemu-kvm libvirt-daemon-system xsltproc diff --git a/dev-docs/workflows/release.md b/dev-docs/workflows/release.md index 12d841dc5..619d28744 100644 --- a/dev-docs/workflows/release.md +++ b/dev-docs/workflows/release.md @@ -46,6 +46,8 @@ Releases should be performed using [the automated release pipeline](https://gith 6. look over the autogenerated draft release. When fixing the changelog, prioritize updating the PR title/labels/description and regenerating the changelog over fixing things in the final changelog. The changelog should be primarily aimed at users. Rule of thumb: first part of the sentence should describe what changed for the user, second part can describe what has been changed to achieve this. 7. in the GitHub release UI, make sure the tag to create on release is set to `$ver`, and the target commit is set to the temporary release branch. 8. publish. +9. follow [post release steps](#post-release-steps). + ### Minor release @@ -78,18 +80,16 @@ Releases should be performed using [the automated release pipeline](https://gith 8. set the Target to `tmp/${ver}` 9. in the GitHub release UI, make sure the tag to create on release is set to `$ver`, and the target commit is set to the temporary release branch. 10. publish. +11. follow [post release steps](#post-release-steps). ## Post release steps -1. Close fixed "known issues" -2. Milestones management - 1. Create a new milestone for the next release - 2. Add the next release manager and an approximate release date to the milestone description - 3. Close the milestone for the release - 4. Move open issues and PRs from closed milestone to next milestone -3. If the release is a minor version release, bump the pre-release version in the `version.txt` file. -4. Update the `fromVersion` in `e2e-test-release.yml` and `e2e-test-weekly.yaml` to the newly released version. To check the current values, run: `grep "fromVersion: \[.*\]" -R .github`. -5. Reset `UpgradeRequiresIAMMigration` in [`iamupgrade.go`](https://github.com/edgelesssys/constellation/blob/a88a731576184e3c5ee8527741c4a0cdaa4e9b24/cli/internal/cloudcmd/iamupgrade.go#L23). +1. Publish the [provider release](https://github.com/edgelesssys/terraform-provider-constellation/releases) +2. Merge the automated post release PR +3. Publish the [AWS and Azure marketplace images](./marketplace-publishing.md). +4. Close fixed "known issues" +5. Move open issues and PRs from this release's closed milestone to next milestone +6. Reset `UpgradeRequiresIAMMigration` in [`iamupgrade.go`](https://github.com/edgelesssys/constellation/blob/a88a731576184e3c5ee8527741c4a0cdaa4e9b24/cli/internal/cloudcmd/iamupgrade.go#L23). ## Troubleshooting: Pipeline cleanup diff --git a/dev-docs/workflows/security-patch.md b/dev-docs/workflows/security-patch.md new file mode 100644 index 000000000..468c9b526 --- /dev/null +++ b/dev-docs/workflows/security-patch.md @@ -0,0 +1,62 @@ +# Security Patch Workflow + +This document describes how to patch vulnerabilities in Constellation. + +## Guiding Principles + +* Constellation vulnerabilities and security patches must be shared on need-to-know basis. +* A vulnerability is only fixed if a patch exists for all supported versions. +* Affected users must be informed about vulnerabilities affecting their setups. +* Vulnerabilities in Constellation should be fixed as quickly as possible. + +## Vulnerability Report + +Someone found a vulnerability in Constellation. +If they followed [SECURITY.md](/SECURITY.md), a Github Security Advisory (GHSA) might already exist. +Otherwise, now is the time to [create a draft](https://github.com/edgelesssys/constellation/security/advisories/new). +Make sure that the GHSA includes a problem statement that gives sufficient context and add domain experts as collaborators. + +## Mitigation + +Investigate possible mitigations for the vulnerability that don't require a patch release. +Such mitigations could include additional firewall settings, manual cluster configuration changes, etc. +Add all reasonable mitigation instructions to the GHSA. + +If the vulnerability has already been disclosed publicly, the GHSA should also be disclosed at this stage. +Add an ETA for a patch release and proceed with [disclosure steps](#disclosing-the-ghsa). + +## Fix + +The first step towards fixing the vulnerability is to assess the amount of work required. +Use this estimate to propose a target release date and inform the product manager about impact and timeline. +The product manager will notify customers of the upcoming patch release. + +Sometimes a fix can be developed quickly for `main`, but correctly backporting it takes more time. +It may also happen that a proposed fix needs substantial work before merging. +In order to avoid premature disclosure of the vulnerability, while still allowing for collaboration, we use the GHSA's temporary repository. + +1. On the drafted GHSA, create a temporary repository to work on a fix. +1. Develop a fix on a local branch, targeting `main`. +1. Manually run static checks (unit tests, linters, etc.). +1. Push the branch to the temporary fork and open a PR. + This is necessary because the fork can't run Github Actions. +1. Solicit and incorporate feedback on the PR. +1. Manually test a fixed version, possibly including upgrade tests. + +When the PR is ready, cherry-pick its commits to your local version of the release branch. +Repeat the steps above, but target the PR at the corresponding release branch. + +Once PRs are ready for all necessary patches, hit the merge button on the GHSA. +This will merge all PRs, but the GHSA will stay in draft mode for now. + +## Disclosing the GHSA + +The following steps need to be performed by a repository admin. + +1. Ensure that the GHSA text is in shape for publication. + In particular, look out for any empty sections and placeholder text. +1. Fill in the `Affected Versions` and `Patched Versions` fields. +1. Check that the severity setting is accurate. +1. Credit external collaborators, e.g. by @-mention. +1. Hit the `Publish Advisory` button. +1. Notify the product manager to share patch and advisory with customers. diff --git a/dev-docs/workflows/terraform-provider.md b/dev-docs/workflows/terraform-provider.md index 5b1e37b73..3dff8e290 100644 --- a/dev-docs/workflows/terraform-provider.md +++ b/dev-docs/workflows/terraform-provider.md @@ -4,8 +4,25 @@ This document explains the basic ways of working with the [Constellation Terrafo ## Building the Terraform Provider -The Constellation Terraform provider can be built through Bazel, either via the [`devbuild` target](./build-develop-deploy.md) (recommended), which will create a `terraform` directory -with the provider binary and some utility files in the current working directory, or explicitly via this command: +The Constellation Terraform provider can be built through Bazel. + +Use the all-in-one Target (Recommended): + +The [`devbuild` target](./build-develop-deploy.md), will create a `terraform` directory +with the provider binary and some utility files in the dedicated local Terraform registry directory. + +```bash +bazel run //:devbuild +``` + +> [!IMPORTANT] when making changes on the provider without a commit, subsequent applies will fail due to the changed binary hash. To solve this, in your Terraform directory run: +> +> ```bash +> rm .terraform.lock.hcl +> terraform init +> ``` + +Only build: ```bash bazel build //terraform-provider-constellation:tf_provider diff --git a/dev-docs/workflows/upgrade-kubernetes.md b/dev-docs/workflows/upgrade-kubernetes.md index 809a837d5..99df23055 100644 --- a/dev-docs/workflows/upgrade-kubernetes.md +++ b/dev-docs/workflows/upgrade-kubernetes.md @@ -27,6 +27,21 @@ curl -qL https://mcr.microsoft.com/v2/oss/kubernetes/azure-cloud-node-manager/ta Normally renovate will handle the upgrading of Kubernetes dependencies. +## Update e2e tests + +Run the following script to update the k8s versions used in the e2e workflows, adjusting the versions to what you're upgrading to. + +```sh +next=v1.33 +current=v1.32 +old=v1.31 +oldold=v1.30 +sed -i -e "s/$current/$next/g" -e "s/$old/$current/g" -e "s/$oldold/$old/g" \ + .github/workflows/e2e-test-daily.yml \ + .github/workflows/e2e-test-weekly.yml \ + .github/workflows/e2e-test-release.yml +``` + ## Test the new Kubernetes version - Setup a Constellation cluster using the new image with the new bootstrapper binary and check if Kubernetes is deployed successfully. diff --git a/disk-mapper/cmd/BUILD.bazel b/disk-mapper/cmd/BUILD.bazel index 1d3b1d565..4db0736b3 100644 --- a/disk-mapper/cmd/BUILD.bazel +++ b/disk-mapper/cmd/BUILD.bazel @@ -30,7 +30,6 @@ go_library( "//internal/logger", "//internal/role", "@com_github_spf13_afero//:afero", - "@org_uber_go_zap//:zap", ], ) diff --git a/disk-mapper/cmd/main.go b/disk-mapper/cmd/main.go index 415b1c56c..8d4e8eee7 100644 --- a/disk-mapper/cmd/main.go +++ b/disk-mapper/cmd/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -9,10 +9,14 @@ package main import ( "context" "flag" + "fmt" "io" + "log/slog" + "log/syslog" "net" "os" "path/filepath" + "time" "github.com/edgelesssys/constellation/v2/disk-mapper/internal/diskencryption" "github.com/edgelesssys/constellation/v2/disk-mapper/internal/recoveryserver" @@ -35,7 +39,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/spf13/afero" - "go.uber.org/zap" ) const ( @@ -47,22 +50,39 @@ const ( ) func main() { + runErr := run() + if runErr == nil { + return + } + syslogWriter, err := syslog.New(syslog.LOG_EMERG|syslog.LOG_KERN, "disk-mapper") + if err != nil { + os.Exit(1) + } + _ = syslogWriter.Err(runErr.Error()) + _ = syslogWriter.Emerg("disk-mapper has failed. In most cases, this is due to a misconfiguration or transient error with the infrastructure.") + time.Sleep(time.Minute) // sleep to allow the message to be written to syslog and seen by the user + os.Exit(1) +} + +func run() error { csp := flag.String("csp", "", "Cloud Service Provider the image is running on") verbosity := flag.Int("v", 0, logger.CmdLineVerbosityDescription) flag.Parse() - log := logger.New(logger.JSONLog, logger.VerbosityFromInt(*verbosity)) - log.With(zap.String("version", constants.BinaryVersion().String()), zap.String("cloudProvider", *csp)). - Infof("Starting disk-mapper") + log := logger.NewJSONLogger(logger.VerbosityFromInt(*verbosity)) + log.With(slog.String("version", constants.BinaryVersion().String()), slog.String("cloudProvider", *csp)). + Info("Starting disk-mapper") // set up quote issuer for aTLS connections attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant)) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant") + log.With(slog.Any("error", err)).Error("Failed to parse attestation variant") + return err } issuer, err := choose.Issuer(attestVariant, log) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to select issuer") + log.With(slog.Any("error", err)).Error("Failed to select issuer") + return err } // set up metadata API @@ -74,31 +94,37 @@ func main() { // using udev rules, a symlink for our disk is created at /dev/sdb diskPath, err = filepath.EvalSymlinks(awsStateDiskPath) if err != nil { - log.With(zap.Error(err)).Fatalf("Unable to resolve Azure state disk path") + log.With(slog.Any("error", err)).Error("Unable to resolve Azure state disk path") + return err } metadataClient, err = awscloud.New(context.Background()) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to set up AWS metadata client") + log.With(slog.Any("error", err)).Error("Failed to set up AWS metadata client") + return err } case cloudprovider.Azure: diskPath, err = filepath.EvalSymlinks(azureStateDiskPath) if err != nil { - log.With(zap.Error(err)).Fatalf("Unable to resolve Azure state disk path") + log.With(slog.Any("error", err)).Error("Unable to resolve Azure state disk path") + return err } metadataClient, err = azurecloud.New(context.Background()) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to set up Azure metadata client") + log.With(slog.Any("error", err)).Error("Failed to set up Azure metadata client") + return err } case cloudprovider.GCP: diskPath, err = filepath.EvalSymlinks(gcpStateDiskPath) if err != nil { - log.With(zap.Error(err)).Fatalf("Unable to resolve GCP state disk path") + log.With(slog.Any("error", err)).Error("Unable to resolve GCP state disk path") + return err } gcpMeta, err := gcpcloud.New(context.Background()) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create GCP metadata client") + log.With(slog.Any("error", err)).Error(("Failed to create GCP metadata client")) + return err } defer gcpMeta.Close() metadataClient = gcpMeta @@ -107,7 +133,8 @@ func main() { diskPath = openstackStateDiskPath metadataClient, err = openstack.New(context.Background()) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to create OpenStack metadata client") + log.With(slog.Any("error", err)).Error(("Failed to create OpenStack metadata client")) + return err } case cloudprovider.QEMU: @@ -115,13 +142,15 @@ func main() { metadataClient = qemucloud.New() default: - log.Fatalf("CSP %s is not supported by Constellation", *csp) + log.Error(fmt.Sprintf("CSP %s is not supported by Constellation", *csp)) + return err } // initialize device mapper mapper, free, err := diskencryption.New(diskPath, log) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to initialize device mapper") + log.With(slog.Any("error", err)).Error(("Failed to initialize device mapper")) + return err } defer free() @@ -133,7 +162,7 @@ func main() { } } setupManger := setup.New( - log.Named("setupManager"), + log.WithGroup("setupManager"), *csp, diskPath, afero.Afero{Fs: afero.NewOsFs()}, @@ -143,7 +172,8 @@ func main() { ) if err := setupManger.LogDevices(); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to log devices") + log.With(slog.Any("error", err)).Error(("Failed to log devices")) + return err } // prepare the state disk @@ -152,21 +182,22 @@ func main() { var self metadata.InstanceMetadata self, err = metadataClient.Self(context.Background()) if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to get self metadata") + log.With(slog.Any("error", err)).Error(("Failed to get self metadata")) + return err } rejoinClient := rejoinclient.New( dialer.New(issuer, nil, &net.Dialer{}), self, metadataClient, - log.Named("rejoinClient"), + log.WithGroup("rejoinClient"), ) // set up recovery server if control-plane node var recoveryServer setup.RecoveryServer if self.Role == role.ControlPlane { - recoveryServer = recoveryserver.New(issuer, kmssetup.KMS, log.Named("recoveryServer")) + recoveryServer = recoveryserver.New(issuer, kmssetup.KMS, log.WithGroup("recoveryServer")) } else { - recoveryServer = recoveryserver.NewStub(log.Named("recoveryServer")) + recoveryServer = recoveryserver.NewStub(log.WithGroup("recoveryServer")) } err = setupManger.PrepareExistingDisk(setup.NewNodeRecoverer(recoveryServer, rejoinClient)) @@ -174,6 +205,8 @@ func main() { err = setupManger.PrepareNewDisk() } if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to prepare state disk") + log.With(slog.Any("error", err)).Error(("Failed to prepare state disk")) + return err } + return nil } diff --git a/disk-mapper/internal/diskencryption/BUILD.bazel b/disk-mapper/internal/diskencryption/BUILD.bazel index 84acaa6c1..452c846de 100644 --- a/disk-mapper/internal/diskencryption/BUILD.bazel +++ b/disk-mapper/internal/diskencryption/BUILD.bazel @@ -5,9 +5,5 @@ go_library( srcs = ["diskencryption.go"], importpath = "github.com/edgelesssys/constellation/v2/disk-mapper/internal/diskencryption", visibility = ["//disk-mapper:__subpackages__"], - deps = [ - "//internal/cryptsetup", - "//internal/logger", - "@org_uber_go_zap//:zap", - ], + deps = ["//internal/cryptsetup"], ) diff --git a/disk-mapper/internal/diskencryption/diskencryption.go b/disk-mapper/internal/diskencryption/diskencryption.go index f6d25a694..c642bb158 100644 --- a/disk-mapper/internal/diskencryption/diskencryption.go +++ b/disk-mapper/internal/diskencryption/diskencryption.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -15,22 +15,21 @@ package diskencryption import ( "fmt" + "log/slog" "time" "github.com/edgelesssys/constellation/v2/internal/cryptsetup" - "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" ) // DiskEncryption handles actions for formatting and mapping crypt devices. type DiskEncryption struct { device cryptDevice devicePath string - log *logger.Logger + log *slog.Logger } // New creates a new crypt device for the device at path. -func New(path string, log *logger.Logger) (*DiskEncryption, func(), error) { +func New(path string, log *slog.Logger) (*DiskEncryption, func(), error) { device := cryptsetup.New() _, err := device.Init(path) if err != nil { @@ -101,7 +100,7 @@ func (d *DiskEncryption) UnmapDisk(target string) error { func (d *DiskEncryption) Wipe(blockWipeSize int) error { logProgress := func(size, offset uint64) { prog := (float64(offset) / float64(size)) * 100 - d.log.With(zap.String("progress", fmt.Sprintf("%.2f%%", prog))).Infof("Wiping disk") + d.log.With(slog.String("progress", fmt.Sprintf("%.2f%%", prog))).Info("Wiping disk") } start := time.Now() @@ -109,7 +108,7 @@ func (d *DiskEncryption) Wipe(blockWipeSize int) error { if err := d.device.Wipe("integrity", blockWipeSize, 0, logProgress, 30*time.Second); err != nil { return fmt.Errorf("wiping disk: %w", err) } - d.log.With(zap.Duration("duration", time.Since(start))).Infof("Wiping disk successful") + d.log.With(slog.Duration("duration", time.Since(start))).Info("Wiping disk successful") return nil } diff --git a/disk-mapper/internal/recoveryserver/BUILD.bazel b/disk-mapper/internal/recoveryserver/BUILD.bazel index ce81e8c39..fc1c24a53 100644 --- a/disk-mapper/internal/recoveryserver/BUILD.bazel +++ b/disk-mapper/internal/recoveryserver/BUILD.bazel @@ -14,10 +14,9 @@ go_library( "//internal/grpc/grpclog", "//internal/kms/kms", "//internal/logger", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", - "@org_uber_go_zap//:zap", ], ) diff --git a/disk-mapper/internal/recoveryserver/recoveryserver.go b/disk-mapper/internal/recoveryserver/recoveryserver.go index d4db3b787..5852b859b 100644 --- a/disk-mapper/internal/recoveryserver/recoveryserver.go +++ b/disk-mapper/internal/recoveryserver/recoveryserver.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -17,6 +17,7 @@ package recoveryserver import ( "context" + "log/slog" "net" "sync" @@ -27,7 +28,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/grpc/grpclog" "github.com/edgelesssys/constellation/v2/internal/kms/kms" "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -45,13 +45,13 @@ type RecoveryServer struct { grpcServer server factory kmsFactory - log *logger.Logger + log *slog.Logger recoverproto.UnimplementedAPIServer } // New returns a new RecoveryServer. -func New(issuer atls.Issuer, factory kmsFactory, log *logger.Logger) *RecoveryServer { +func New(issuer atls.Issuer, factory kmsFactory, log *slog.Logger) *RecoveryServer { server := &RecoveryServer{ log: log, factory: factory, @@ -59,7 +59,7 @@ func New(issuer atls.Issuer, factory kmsFactory, log *logger.Logger) *RecoverySe grpcServer := grpc.NewServer( grpc.Creds(atlscredentials.New(issuer, nil)), - log.Named("gRPC").GetServerStreamInterceptor(), + logger.GetServerStreamInterceptor(logger.GRPCLogger(log)), ) recoverproto.RegisterAPIServer(grpcServer, server) @@ -72,7 +72,7 @@ func New(issuer atls.Issuer, factory kmsFactory, log *logger.Logger) *RecoverySe // The server will shut down when the call is successful and the keys are returned. // Additionally, the server can be shutdown by canceling the context. func (s *RecoveryServer) Serve(ctx context.Context, listener net.Listener, diskUUID string) (diskKey, measurementSecret []byte, err error) { - s.log.Infof("Starting RecoveryServer") + s.log.Info("Starting RecoveryServer") s.diskUUID = diskUUID recoveryDone := make(chan struct{}, 1) var serveErr error @@ -89,7 +89,7 @@ func (s *RecoveryServer) Serve(ctx context.Context, listener net.Listener, diskU for { select { case <-ctx.Done(): - s.log.Infof("Context canceled, shutting down server") + s.log.Info("Context canceled, shutting down server") s.grpcServer.GracefulStop() return nil, nil, ctx.Err() case <-recoveryDone: @@ -105,9 +105,9 @@ func (s *RecoveryServer) Serve(ctx context.Context, listener net.Listener, diskU func (s *RecoveryServer) Recover(ctx context.Context, req *recoverproto.RecoverMessage) (*recoverproto.RecoverResponse, error) { s.mux.Lock() defer s.mux.Unlock() - log := s.log.With(zap.String("peer", grpclog.PeerAddrFromContext(ctx))) + log := s.log.With(slog.String("peer", grpclog.PeerAddrFromContext(ctx))) - log.Infof("Received recover call") + log.Info("Received recover call") cloudKms, err := s.factory(ctx, req.StorageUri, req.KmsUri) if err != nil { @@ -124,7 +124,7 @@ func (s *RecoveryServer) Recover(ctx context.Context, req *recoverproto.RecoverM } s.stateDiskKey = stateDiskKey s.measurementSecret = measurementSecret - log.Infof("Received state disk key and measurement secret, shutting down server") + log.Info("Received state disk key and measurement secret, shutting down server") go s.grpcServer.GracefulStop() return &recoverproto.RecoverResponse{}, nil @@ -132,18 +132,18 @@ func (s *RecoveryServer) Recover(ctx context.Context, req *recoverproto.RecoverM // StubServer implements the RecoveryServer interface but does not actually start a server. type StubServer struct { - log *logger.Logger + log *slog.Logger } // NewStub returns a new stubbed RecoveryServer. // We use this to avoid having to start a server for worker nodes, since they don't require manual recovery. -func NewStub(log *logger.Logger) *StubServer { +func NewStub(log *slog.Logger) *StubServer { return &StubServer{log: log} } // Serve waits until the context is canceled and returns nil. func (s *StubServer) Serve(ctx context.Context, _ net.Listener, _ string) ([]byte, []byte, error) { - s.log.Infof("Running as worker node, skipping recovery server") + s.log.Info("Running as worker node, skipping recovery server") <-ctx.Done() return nil, nil, ctx.Err() } diff --git a/disk-mapper/internal/recoveryserver/recoveryserver_test.go b/disk-mapper/internal/recoveryserver/recoveryserver_test.go index 4673c1137..5676de7f0 100644 --- a/disk-mapper/internal/recoveryserver/recoveryserver_test.go +++ b/disk-mapper/internal/recoveryserver/recoveryserver_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package recoveryserver @@ -29,6 +29,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -39,7 +40,7 @@ func TestServe(t *testing.T) { server := New(atls.NewFakeIssuer(variant.Dummy{}), newStubKMS(nil, nil), log) dialer := testdialer.NewBufconnDialer() listener := dialer.GetListener("192.0.2.1:1234") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) var wg sync.WaitGroup // Serve method returns when context is canceled @@ -61,7 +62,7 @@ func TestServe(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _, _, err := server.Serve(context.Background(), listener, uuid) + _, _, err := server.Serve(t.Context(), listener, uuid) assert.NoError(err) }() time.Sleep(100 * time.Millisecond) @@ -69,7 +70,7 @@ func TestServe(t *testing.T) { wg.Wait() // Serve method returns an error when serving is unsuccessful - _, _, err := server.Serve(context.Background(), listener, uuid) + _, _, err := server.Serve(t.Context(), listener, uuid) assert.Error(err) } @@ -103,7 +104,7 @@ func TestRecover(t *testing.T) { assert := assert.New(t) require := require.New(t) - ctx := context.Background() + ctx := t.Context() serverUUID := "uuid" server := New(atls.NewFakeIssuer(variant.Dummy{}), tc.factory, logger.NewTest(t)) netDialer := testdialer.NewBufconnDialer() @@ -122,7 +123,7 @@ func TestRecover(t *testing.T) { diskKey, measurementSecret, serveErr = server.Serve(serveCtx, listener, serverUUID) }() - conn, err := dialer.New(nil, nil, netDialer).Dial(ctx, "192.0.2.1:1234") + conn, err := dialer.New(nil, nil, netDialer).Dial("192.0.2.1:1234") require.NoError(err) defer conn.Close() @@ -146,7 +147,7 @@ func TestRecover(t *testing.T) { } func newStubKMS(setupErr, getDEKErr error) kmsFactory { - return func(ctx context.Context, storageURI string, kmsURI string) (kms.CloudKMS, error) { + return func(_ context.Context, _ string, _ string) (kms.CloudKMS, error) { if setupErr != nil { return nil, setupErr } diff --git a/disk-mapper/internal/rejoinclient/BUILD.bazel b/disk-mapper/internal/rejoinclient/BUILD.bazel index b7bae0f1c..77238f692 100644 --- a/disk-mapper/internal/rejoinclient/BUILD.bazel +++ b/disk-mapper/internal/rejoinclient/BUILD.bazel @@ -9,12 +9,10 @@ go_library( deps = [ "//internal/cloud/metadata", "//internal/constants", - "//internal/logger", "//internal/role", "//joinservice/joinproto", "@io_k8s_utils//clock", - "@org_golang_google_grpc//:go_default_library", - "@org_uber_go_zap//:zap", + "@org_golang_google_grpc//:grpc", ], ) @@ -33,7 +31,7 @@ go_test( "//joinservice/joinproto", "@com_github_stretchr_testify//assert", "@io_k8s_utils//clock/testing", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_uber_go_goleak//:goleak", ], ) diff --git a/disk-mapper/internal/rejoinclient/rejoinclient.go b/disk-mapper/internal/rejoinclient/rejoinclient.go index 4f50b0b41..bbd511971 100644 --- a/disk-mapper/internal/rejoinclient/rejoinclient.go +++ b/disk-mapper/internal/rejoinclient/rejoinclient.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -15,16 +15,15 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" "time" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" "github.com/edgelesssys/constellation/v2/joinservice/joinproto" - "go.uber.org/zap" "google.golang.org/grpc" "k8s.io/utils/clock" ) @@ -47,12 +46,12 @@ type RejoinClient struct { dialer grpcDialer metadataAPI metadataAPI - log *logger.Logger + log *slog.Logger } // New returns a new RejoinClient. func New(dial grpcDialer, nodeInfo metadata.InstanceMetadata, - meta metadataAPI, log *logger.Logger, + meta metadataAPI, log *slog.Logger, ) *RejoinClient { return &RejoinClient{ nodeInfo: nodeInfo, @@ -70,22 +69,22 @@ func New(dial grpcDialer, nodeInfo metadata.InstanceMetadata, // from the metadata API and send rejoin requests to them. // The function returns after a successful rejoin request has been performed. func (c *RejoinClient) Start(ctx context.Context, diskUUID string) (diskKey, measurementSecret []byte) { - c.log.Infof("Starting RejoinClient") + c.log.Info("Starting RejoinClient") c.diskUUID = diskUUID ticker := c.clock.NewTicker(c.interval) defer ticker.Stop() - defer c.log.Infof("RejoinClient stopped") + defer c.log.Info("RejoinClient stopped") for { endpoints, err := c.getJoinEndpoints() if err != nil { - c.log.With(zap.Error(err)).Errorf("Failed to get control-plane endpoints") + c.log.With(slog.Any("error", err)).Error("Failed to get control-plane endpoints") } else { - c.log.With(zap.Strings("endpoints", endpoints)).Infof("Received list with JoinService endpoints") + c.log.With(slog.Any("endpoints", endpoints)).Info("Received list with JoinService endpoints") diskKey, measurementSecret, err = c.tryRejoinWithAvailableServices(ctx, endpoints) if err == nil { - c.log.Infof("Successfully retrieved rejoin ticket") + c.log.Info("Successfully retrieved rejoin ticket") return diskKey, measurementSecret } } @@ -101,12 +100,12 @@ func (c *RejoinClient) Start(ctx context.Context, diskUUID string) (diskKey, mea // tryRejoinWithAvailableServices tries sending rejoin requests to the available endpoints. func (c *RejoinClient) tryRejoinWithAvailableServices(ctx context.Context, endpoints []string) (diskKey, measurementSecret []byte, err error) { for _, endpoint := range endpoints { - c.log.With(zap.String("endpoint", endpoint)).Infof("Requesting rejoin ticket") + c.log.With(slog.String("endpoint", endpoint)).Info("Requesting rejoin ticket") rejoinTicket, err := c.requestRejoinTicket(endpoint) if err == nil { return rejoinTicket.StateDiskKey, rejoinTicket.MeasurementSecret, nil } - c.log.With(zap.Error(err), zap.String("endpoint", endpoint)).Warnf("Failed to rejoin on endpoint") + c.log.With(slog.Any("error", err), slog.String("endpoint", endpoint)).Warn("Failed to rejoin on endpoint") // stop requesting additional endpoints if the context is done select { @@ -115,7 +114,7 @@ func (c *RejoinClient) tryRejoinWithAvailableServices(ctx context.Context, endpo default: } } - c.log.Errorf("Failed to rejoin on all endpoints") + c.log.Error("Failed to rejoin on all endpoints") return nil, nil, errors.New("failed to join on all endpoints") } @@ -124,7 +123,7 @@ func (c *RejoinClient) requestRejoinTicket(endpoint string) (*joinproto.IssueRej ctx, cancel := c.timeoutCtx() defer cancel() - conn, err := c.dialer.Dial(ctx, endpoint) + conn, err := c.dialer.Dial(endpoint) if err != nil { return nil, err } @@ -186,7 +185,7 @@ func (c *RejoinClient) timeoutCtx() (context.Context, context.CancelFunc) { } type grpcDialer interface { - Dial(ctx context.Context, target string) (*grpc.ClientConn, error) + Dial(target string) (*grpc.ClientConn, error) } type metadataAPI interface { diff --git a/disk-mapper/internal/rejoinclient/rejoinclient_test.go b/disk-mapper/internal/rejoinclient/rejoinclient_test.go index e78be25a3..51e3a6bd1 100644 --- a/disk-mapper/internal/rejoinclient/rejoinclient_test.go +++ b/disk-mapper/internal/rejoinclient/rejoinclient_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package rejoinclient @@ -30,7 +30,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestStartCancel(t *testing.T) { @@ -71,7 +71,7 @@ func TestStartCancel(t *testing.T) { go rejoinServer.Serve(listener) defer rejoinServer.GracefulStop() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) var wg sync.WaitGroup wg.Add(1) @@ -294,7 +294,7 @@ func TestStart(t *testing.T) { client := New(dialer, tc.nodeInfo, meta, logger.NewTest(t)) - passphrase, secret := client.Start(context.Background(), "uuid") + passphrase, secret := client.Start(t.Context(), "uuid") assert.Equal(diskKey, passphrase) assert.Equal(measurementSecret, secret) }) diff --git a/disk-mapper/internal/setup/BUILD.bazel b/disk-mapper/internal/setup/BUILD.bazel index 035d57304..6250ff87d 100644 --- a/disk-mapper/internal/setup/BUILD.bazel +++ b/disk-mapper/internal/setup/BUILD.bazel @@ -20,10 +20,8 @@ go_library( "//internal/constants", "//internal/crypto", "//internal/file", - "//internal/logger", "//internal/nodestate", "@com_github_spf13_afero//:afero", - "@org_uber_go_zap//:zap", ], ) diff --git a/disk-mapper/internal/setup/interface.go b/disk-mapper/internal/setup/interface.go index 50bd008a7..6f4e02c86 100644 --- a/disk-mapper/internal/setup/interface.go +++ b/disk-mapper/internal/setup/interface.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package setup diff --git a/disk-mapper/internal/setup/mount_cross.go b/disk-mapper/internal/setup/mount_cross.go index 271a467d0..1c8015ee8 100644 --- a/disk-mapper/internal/setup/mount_cross.go +++ b/disk-mapper/internal/setup/mount_cross.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package setup diff --git a/disk-mapper/internal/setup/mount_linux.go b/disk-mapper/internal/setup/mount_linux.go index d3ee2d229..f0ba7dff0 100644 --- a/disk-mapper/internal/setup/mount_linux.go +++ b/disk-mapper/internal/setup/mount_linux.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package setup diff --git a/disk-mapper/internal/setup/setup.go b/disk-mapper/internal/setup/setup.go index 312880713..47b82a348 100644 --- a/disk-mapper/internal/setup/setup.go +++ b/disk-mapper/internal/setup/setup.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -17,6 +17,7 @@ import ( "errors" "fmt" "io/fs" + "log/slog" "net" "os" "path/filepath" @@ -31,10 +32,8 @@ import ( "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/crypto" "github.com/edgelesssys/constellation/v2/internal/file" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/nodestate" "github.com/spf13/afero" - "go.uber.org/zap" ) const ( @@ -49,7 +48,7 @@ const ( // Manager handles formatting, mapping, mounting and unmounting of state disks. type Manager struct { - log *logger.Logger + log *slog.Logger csp string diskPath string fs afero.Afero @@ -60,7 +59,7 @@ type Manager struct { } // New initializes a SetupManager with the given parameters. -func New(log *logger.Logger, csp string, diskPath string, fs afero.Afero, +func New(log *slog.Logger, csp string, diskPath string, fs afero.Afero, mapper DeviceMapper, mounter Mounter, openDevice vtpm.TPMOpenFunc, ) *Manager { return &Manager{ @@ -77,15 +76,15 @@ func New(log *logger.Logger, csp string, diskPath string, fs afero.Afero, // PrepareExistingDisk requests and waits for a decryption key to remap the encrypted state disk. // Once the disk is mapped, the function taints the node as initialized by updating it's PCRs. -func (s *Manager) PrepareExistingDisk(recover RecoveryDoer) error { +func (s *Manager) PrepareExistingDisk(recoverer RecoveryDoer) error { uuid, err := s.mapper.DiskUUID() if err != nil { return err } - s.log.With(zap.String("uuid", uuid)).Infof("Preparing existing state disk") + s.log.With(slog.String("uuid", uuid)).Info("Preparing existing state disk") endpoint := net.JoinHostPort("0.0.0.0", strconv.Itoa(constants.RecoveryPort)) - passphrase, measurementSecret, err := recover.Do(uuid, endpoint) + passphrase, measurementSecret, err := recoverer.Do(uuid, endpoint) if err != nil { return fmt.Errorf("failed to perform recovery: %w", err) } @@ -128,7 +127,7 @@ func (s *Manager) PrepareExistingDisk(recover RecoveryDoer) error { // PrepareNewDisk prepares an instances state disk by formatting the disk as a LUKS device using a random passphrase. func (s *Manager) PrepareNewDisk() error { uuid, _ := s.mapper.DiskUUID() - s.log.With(zap.String("uuid", uuid)).Infof("Preparing new state disk") + s.log.With(slog.String("uuid", uuid)).Info("Preparing new state disk") // generate and save temporary passphrase passphrase := make([]byte, crypto.RNGLengthDefault) @@ -192,12 +191,12 @@ func (s *Manager) LogDevices() error { devices = append(devices, fileInfo) } - s.log.Infof("List of all available block devices and partitions:") + s.log.Info("List of all available block devices and partitions:") for _, device := range devices { var stat syscall.Statfs_t dev := "/dev/" + device.Name() if err := syscall.Statfs(dev, &stat); err != nil { - s.log.With(zap.Error(err)).Errorf("failed to statfs %s", dev) + s.log.With(slog.Any("error", err)).Error(fmt.Sprintf("failed to statfs %s", dev)) continue } @@ -206,7 +205,7 @@ func (s *Manager) LogDevices() error { free := stat.Bfree * uint64(stat.Bsize) avail := stat.Bavail * uint64(stat.Bsize) - s.log.Infof( + s.log.Info(fmt.Sprintf( "Name: %-15s, Size: %-10d, Mode: %s, ModTime: %s, Size = %-10d, Free = %-10d, Available = %-10d\n", dev, device.Size(), @@ -214,7 +213,7 @@ func (s *Manager) LogDevices() error { device.ModTime(), size, free, - avail) + avail)) } return nil } diff --git a/disk-mapper/internal/setup/setup_test.go b/disk-mapper/internal/setup/setup_test.go index e6720a588..a21416101 100644 --- a/disk-mapper/internal/setup/setup_test.go +++ b/disk-mapper/internal/setup/setup_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package setup @@ -28,7 +28,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestPrepareExistingDisk(t *testing.T) { diff --git a/disk-mapper/internal/systemd/systemd.go b/disk-mapper/internal/systemd/systemd.go index 843f26c7b..7520c39fe 100644 --- a/disk-mapper/internal/systemd/systemd.go +++ b/disk-mapper/internal/systemd/systemd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package systemd configures systemd units for encrypted volumes. diff --git a/disk-mapper/internal/systemd/systemd_test.go b/disk-mapper/internal/systemd/systemd_test.go index c62d46dcc..7accd2f58 100644 --- a/disk-mapper/internal/systemd/systemd_test.go +++ b/disk-mapper/internal/systemd/systemd_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package systemd @@ -14,7 +14,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestConfigureUnit(t *testing.T) { diff --git a/disk-mapper/internal/test/BUILD.bazel b/disk-mapper/internal/test/BUILD.bazel index 489a2b89f..38e3ac89c 100644 --- a/disk-mapper/internal/test/BUILD.bazel +++ b/disk-mapper/internal/test/BUILD.bazel @@ -29,9 +29,8 @@ go_test( "@com_github_martinjungblut_go_cryptsetup//:go-cryptsetup", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", + "@io_bazel_rules_go//go/runfiles", "@org_uber_go_goleak//:goleak", - "@org_uber_go_zap//zapcore", ], "@io_bazel_rules_go//go/platform:linux": [ "//disk-mapper/internal/diskencryption", @@ -40,9 +39,8 @@ go_test( "@com_github_martinjungblut_go_cryptsetup//:go-cryptsetup", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", + "@io_bazel_rules_go//go/runfiles", "@org_uber_go_goleak//:goleak", - "@org_uber_go_zap//zapcore", ], "//conditions:default": [], }), diff --git a/disk-mapper/internal/test/benchmark_test.go b/disk-mapper/internal/test/benchmark_test.go index 6fc92a284..419b79608 100644 --- a/disk-mapper/internal/test/benchmark_test.go +++ b/disk-mapper/internal/test/benchmark_test.go @@ -3,20 +3,20 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package integration import ( "fmt" + "log/slog" "math" "testing" "github.com/edgelesssys/constellation/v2/disk-mapper/internal/diskencryption" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/martinjungblut/go-cryptsetup" - "go.uber.org/zap/zapcore" ) func BenchmarkMapper(b *testing.B) { @@ -39,7 +39,7 @@ func BenchmarkMapper(b *testing.B) { } passphrase := "benchmark" - mapper, free, err := diskencryption.New(testPath, logger.New(logger.PlainLog, zapcore.InfoLevel)) + mapper, free, err := diskencryption.New(testPath, logger.NewTextLogger(slog.LevelInfo)) if err != nil { b.Fatal("Failed to create mapper:", err) } diff --git a/disk-mapper/internal/test/integration_test.go b/disk-mapper/internal/test/integration_test.go index 7cfdc4b96..364c97088 100644 --- a/disk-mapper/internal/test/integration_test.go +++ b/disk-mapper/internal/test/integration_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package integration @@ -12,6 +12,7 @@ import ( "encoding/json" "flag" "fmt" + "log/slog" "os" "os/exec" "path/filepath" @@ -36,7 +37,7 @@ const ( var diskPath = flag.String("disk", "", "Path to the disk to use for the benchmark") -var toolsEnvs []string = []string{"DD", "RM"} +var toolsEnvs = []string{"DD", "RM"} // addToolsToPATH is used to update the PATH to contain necessary tool binaries for // coreutils. @@ -87,6 +88,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) result := m.Run() @@ -102,7 +104,7 @@ func TestMapper(t *testing.T) { require.NoError(setup(1), "failed to setup test disk") defer func() { require.NoError(teardown(), "failed to delete test disk") }() - mapper, free, err := diskencryption.New(devicePath, logger.NewTest(t)) + mapper, free, err := diskencryption.New(devicePath, logger.NewTextLogger(slog.LevelInfo)) require.NoError(err, "failed to initialize crypt device") defer free() diff --git a/disk-mapper/recoverproto/recover.pb.go b/disk-mapper/recoverproto/recover.pb.go index dbf7f817e..cf62209b3 100644 --- a/disk-mapper/recoverproto/recover.pb.go +++ b/disk-mapper/recoverproto/recover.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v4.22.1 +// protoc-gen-go v1.36.6 +// protoc v5.29.1 // source: disk-mapper/recoverproto/recover.proto package recoverproto @@ -15,6 +15,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -25,21 +26,18 @@ const ( ) type RecoverMessage struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + KmsUri string `protobuf:"bytes,3,opt,name=kms_uri,json=kmsUri,proto3" json:"kms_uri,omitempty"` + StorageUri string `protobuf:"bytes,4,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"` unknownFields protoimpl.UnknownFields - - KmsUri string `protobuf:"bytes,3,opt,name=kms_uri,json=kmsUri,proto3" json:"kms_uri,omitempty"` - StorageUri string `protobuf:"bytes,4,opt,name=storage_uri,json=storageUri,proto3" json:"storage_uri,omitempty"` + sizeCache protoimpl.SizeCache } func (x *RecoverMessage) Reset() { *x = RecoverMessage{} - if protoimpl.UnsafeEnabled { - mi := &file_disk_mapper_recoverproto_recover_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_disk_mapper_recoverproto_recover_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RecoverMessage) String() string { @@ -50,7 +48,7 @@ func (*RecoverMessage) ProtoMessage() {} func (x *RecoverMessage) ProtoReflect() protoreflect.Message { mi := &file_disk_mapper_recoverproto_recover_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -80,18 +78,16 @@ func (x *RecoverMessage) GetStorageUri() string { } type RecoverResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RecoverResponse) Reset() { *x = RecoverResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_disk_mapper_recoverproto_recover_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_disk_mapper_recoverproto_recover_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RecoverResponse) String() string { @@ -102,7 +98,7 @@ func (*RecoverResponse) ProtoMessage() {} func (x *RecoverResponse) ProtoReflect() protoreflect.Message { mi := &file_disk_mapper_recoverproto_recover_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -119,43 +115,31 @@ func (*RecoverResponse) Descriptor() ([]byte, []int) { var File_disk_mapper_recoverproto_recover_proto protoreflect.FileDescriptor -var file_disk_mapper_recoverproto_recover_proto_rawDesc = []byte{ - 0x0a, 0x26, 0x64, 0x69, 0x73, 0x6b, 0x2d, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x72, 0x65, - 0x63, 0x6f, 0x76, 0x65, 0x72, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x63, 0x6f, 0x76, - 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x76, 0x65, - 0x72, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x4a, 0x0a, 0x0e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, - 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6b, 0x6d, 0x73, 0x5f, - 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x6d, 0x73, 0x55, 0x72, - 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x69, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, - 0x72, 0x69, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x4f, 0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x48, 0x0a, 0x07, - 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x12, 0x1c, 0x2e, 0x72, 0x65, 0x63, 0x6f, 0x76, 0x65, - 0x72, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1d, 0x2e, 0x72, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, - 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, - 0x32, 0x2f, 0x64, 0x69, 0x73, 0x6b, 0x2d, 0x6d, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x72, 0x65, - 0x63, 0x6f, 0x76, 0x65, 0x72, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, -} +const file_disk_mapper_recoverproto_recover_proto_rawDesc = "" + + "\n" + + "&disk-mapper/recoverproto/recover.proto\x12\frecoverproto\"J\n" + + "\x0eRecoverMessage\x12\x17\n" + + "\akms_uri\x18\x03 \x01(\tR\x06kmsUri\x12\x1f\n" + + "\vstorage_uri\x18\x04 \x01(\tR\n" + + "storageUri\"\x11\n" + + "\x0fRecoverResponse2O\n" + + "\x03API\x12H\n" + + "\aRecover\x12\x1c.recoverproto.RecoverMessage\x1a\x1d.recoverproto.RecoverResponse\"\x00BBZ@github.com/edgelesssys/constellation/v2/disk-mapper/recoverprotob\x06proto3" var ( file_disk_mapper_recoverproto_recover_proto_rawDescOnce sync.Once - file_disk_mapper_recoverproto_recover_proto_rawDescData = file_disk_mapper_recoverproto_recover_proto_rawDesc + file_disk_mapper_recoverproto_recover_proto_rawDescData []byte ) func file_disk_mapper_recoverproto_recover_proto_rawDescGZIP() []byte { file_disk_mapper_recoverproto_recover_proto_rawDescOnce.Do(func() { - file_disk_mapper_recoverproto_recover_proto_rawDescData = protoimpl.X.CompressGZIP(file_disk_mapper_recoverproto_recover_proto_rawDescData) + file_disk_mapper_recoverproto_recover_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_disk_mapper_recoverproto_recover_proto_rawDesc), len(file_disk_mapper_recoverproto_recover_proto_rawDesc))) }) return file_disk_mapper_recoverproto_recover_proto_rawDescData } var file_disk_mapper_recoverproto_recover_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_disk_mapper_recoverproto_recover_proto_goTypes = []interface{}{ +var file_disk_mapper_recoverproto_recover_proto_goTypes = []any{ (*RecoverMessage)(nil), // 0: recoverproto.RecoverMessage (*RecoverResponse)(nil), // 1: recoverproto.RecoverResponse } @@ -174,37 +158,11 @@ func file_disk_mapper_recoverproto_recover_proto_init() { if File_disk_mapper_recoverproto_recover_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_disk_mapper_recoverproto_recover_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RecoverMessage); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_disk_mapper_recoverproto_recover_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RecoverResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_disk_mapper_recoverproto_recover_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_disk_mapper_recoverproto_recover_proto_rawDesc), len(file_disk_mapper_recoverproto_recover_proto_rawDesc)), NumEnums: 0, NumMessages: 2, NumExtensions: 0, @@ -215,7 +173,6 @@ func file_disk_mapper_recoverproto_recover_proto_init() { MessageInfos: file_disk_mapper_recoverproto_recover_proto_msgTypes, }.Build() File_disk_mapper_recoverproto_recover_proto = out.File - file_disk_mapper_recoverproto_recover_proto_rawDesc = nil file_disk_mapper_recoverproto_recover_proto_goTypes = nil file_disk_mapper_recoverproto_recover_proto_depIdxs = nil } diff --git a/docs/.gitignore b/docs/.gitignore index b2db0247a..ef904e99f 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,4 +1,3 @@ node_modules -package-lock.json .docusaurus build/ \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index dea229064..4814729e9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ During edits you can preview your changes using the [`docusaurus`](https://docus ```sh # requires node >=16.14 -npm install +npm ci # Install pinned dependencies npm run build npm run serve ``` diff --git a/docs/docs/_media/concept-constellation.svg b/docs/docs/_media/concept-constellation.svg index caa7f847d..30d32bf6d 100644 --- a/docs/docs/_media/concept-constellation.svg +++ b/docs/docs/_media/concept-constellation.svg @@ -1,35 +1,14 @@ + fill-opacity="1" + id="path10" /> diff --git a/docs/docs/_media/concept-managed.svg b/docs/docs/_media/concept-managed.svg index 718412aad..5645a608f 100644 --- a/docs/docs/_media/concept-managed.svg +++ b/docs/docs/_media/concept-managed.svg @@ -1,35 +1,14 @@ + id="g61"> diff --git a/docs/docs/_media/tcb.svg b/docs/docs/_media/tcb.svg index f692ffd0e..e5bcb5b95 100644 --- a/docs/docs/_media/tcb.svg +++ b/docs/docs/_media/tcb.svg @@ -1,35 +1,14 @@ + x="1.38965" + y="1.38928" + width="1560.67" + height="390.667" + id="rect3" /> diff --git a/docs/docs/architecture/attestation.md b/docs/docs/architecture/attestation.md index 04b85d8ad..9bd157460 100644 --- a/docs/docs/architecture/attestation.md +++ b/docs/docs/architecture/attestation.md @@ -78,15 +78,15 @@ The idea is that Constellation nodes should have verifiable integrity from the C The solution is a verifiable boot chain and an integrity-protected runtime environment. Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. -Outside of CC, it's usually implemented via TPMs. +Outside of CC, this is usually implemented via TPMs. CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. For simplicity, TPM terminology like *PCR* is used in the following. -When a Constellation node image boots inside a CVM, it uses measured boot for all stages and components of the boot chain. +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. This process goes up to the root filesystem. -The root filesystem is mounted read-only with integrity protection, guaranteeing forward integrity. +The root filesystem is mounted read-only with integrity protection. For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. -Any changes to the image will inevitably also change the measured boot's PCR values. +Any changes to the image will inevitably also change the corresponding PCR values. To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. @@ -121,8 +121,40 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +184,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,10 +217,10 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + -Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. +Constellation uses a hypervisor-based vTPM for runtime measurements. The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). @@ -199,14 +231,14 @@ The latter means that the value can be generated offline and compared to the one | PCR | Components | Measured by | Reproducible and verifiable | | ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | -| 0 | Firmware | AWS | No | -| 1 | Firmware | AWS | No | -| 2 | Firmware | AWS | No | -| 3 | Firmware | AWS | No | -| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | -| 5 | Firmware | AWS | No | -| 6 | Firmware | AWS | No | -| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | | 8 | - | - | - | | 9 | initramfs, Kernel command line | Linux Kernel | Yes | | 10 | User space | Linux IMA | No[^1] | @@ -217,16 +249,38 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,16 +302,11 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + -There is no additional configuration available for GCP. - - - - -On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. -An SEV-SNP attestation report is used to establish trust in the VM and it's vTPM. +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. You may customize certain parameters for verification of the attestation statement using the Constellation config file. * TCB versions @@ -275,8 +324,15 @@ You may customize certain parameters for verification of the attestation stateme This is the intermediate certificate for verifying the SEV-SNP report's signature. If it's not specified, the CLI fetches it from the AMD key distribution server. - - + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + ## Cluster attestation @@ -300,32 +356,52 @@ When an initialized node tries to join another cluster, its measurements inevita The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. -## Chain of trust +## Putting it all together -So far, this page described how an entire Constellation cluster can be verified using hardware attestation capabilities and runtime measurements. -The last missing link is how the ground truth in the form of runtime measurements can be securely distributed to the verifying party. +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. -The build process of Constellation images also creates the ground truth runtime measurements. -With every release, Edgeless Systems publishes signed runtime measurements. +### CLI and node images -The CLI executable is also signed by Edgeless Systems. -You can [verify its signature](../workflows/verify-cli.md). +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. -The CLI contains the public key required to verify signed runtime measurements from Edgeless Systems. -When a cluster is [created](../workflows/create.md) or [upgraded](../workflows/upgrade.md), the CLI automatically verifies the measurements for the selected image. +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: -Thus, there's a chain of trust based on cryptographic signatures, which goes from CLI to runtime measurements to images. This is illustrated in the following diagram. +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. ```mermaid flowchart LR - A[Edgeless]-- "signs (cosign)" -->B[CLI] - C[User]-- "verifies (cosign)" -->B[CLI] - B[CLI]-- "contains" -->D["Public Key"] - A[Edgeless]-- "signs" -->E["Runtime measurements"] - D["Public key"]-- "verifies" -->E["Runtime measurements"] - E["Runtime measurements"]-- "verify" -->F["Constellation cluster"] + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] ``` +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + ## References [^1]: Linux IMA produces runtime measurements of user-space binaries. diff --git a/docs/docs/architecture/keys.md b/docs/docs/architecture/keys.md index f2c8c3fba..49821cd0b 100644 --- a/docs/docs/architecture/keys.md +++ b/docs/docs/architecture/keys.md @@ -42,7 +42,6 @@ Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). -Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. ## Storage encryption @@ -105,7 +104,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/docs/architecture/observability.md b/docs/docs/architecture/observability.md index 39018e6b1..0f4daffd4 100644 --- a/docs/docs/architecture/observability.md +++ b/docs/docs/architecture/observability.md @@ -33,11 +33,7 @@ The payload is an actual message emitted from your system along with a metadata ### System logs -Constellation uses cloud logging for events occurring during the early stages of a node's boot process. -These logs include [Bootstrapper](./microservices.md#bootstrapper) events and [state disk UUIDs](../architecture/images.md#state-disk). -You can access the cloud logging [directly via the cloud provider endpoints](../workflows/troubleshooting.md#cloud-logging). - -More detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: diff --git a/docs/docs/architecture/versions.md b/docs/docs/architecture/versions.md index 6ee17873c..9c9aebc52 100644 --- a/docs/docs/architecture/versions.md +++ b/docs/docs/architecture/versions.md @@ -16,6 +16,6 @@ Subsequent Constellation releases drop support for the oldest (and deprecated) K The following Kubernetes versions are currently supported: -* v1.26.11 -* v1.27.8 -* v1.28.4 +* v1.30.14 +* v1.31.11 +* v1.32.7 diff --git a/docs/docs/getting-started/first-steps-local.md b/docs/docs/getting-started/first-steps-local.md index 052d29eae..98f0302de 100644 --- a/docs/docs/getting-started/first-steps-local.md +++ b/docs/docs/getting-started/first-steps-local.md @@ -45,8 +45,8 @@ sudo iptables -P FORWARD ACCEPT ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -74,8 +74,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -145,8 +145,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -199,8 +199,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -211,8 +211,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -240,8 +240,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/docs/getting-started/first-steps.md b/docs/docs/getting-started/first-steps.md index c58d4a0ae..fb8437a06 100644 --- a/docs/docs/getting-started/first-steps.md +++ b/docs/docs/getting-started/first-steps.md @@ -13,70 +13,43 @@ If you encounter any problem with the following steps, make sure to use the [lat ## Create a cluster -1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. - - - - - ```bash - constellation config generate azure - ``` - - - - - - ```bash - constellation config generate gcp - ``` - - - - + + ```bash constellation config generate aws ``` - + + - + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + 2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). - - - - - ```bash - constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config - ``` - - This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. - - Note that CVMs are currently only supported in a few regions, check [Azure's products available by region](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). These are: - * `westus` - * `eastus` - * `northeurope` - * `westeurope` - * `southeastasia` - - - - - - ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --update-config - ``` - - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. - - Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - - - - + + ```bash constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config @@ -103,8 +76,55 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --prefix=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + :::caution + + `stackitProjectID` refers to the ID of your STACKIT project. The STACKIT portal also shows the OpenStack ID that's associated with your project in some places. Make sure you insert the STACKIT project ID in the `constellation-conf.yaml` file. It's of the format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. + + ::: + + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/docs/getting-started/install.md b/docs/docs/getting-started/install.md index 8a2313e2d..f072407d8 100644 --- a/docs/docs/getting-started/install.md +++ b/docs/docs/getting-started/install.md @@ -1,25 +1,29 @@ # Installation and setup -Constellation runs entirely in your cloud environment and can be controlled via a dedicated command-line interface (CLI). - -The following guides you through the steps of installing the CLI on your machine, verifying it, and connecting it to your cloud service provider (CSP). +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). ## Prerequisites Make sure the following requirements are met: -- Your machine is running Linux or macOS -- You have admin rights on your machine -- [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT ## Install the Constellation CLI +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +39,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +56,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +74,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +92,31 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -100,7 +124,7 @@ The CLI supports autocompletion for various shells. To set it up, run `constella ## Set up cloud credentials -The CLI makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. :::tip If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. @@ -108,11 +132,63 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] + +* `Microsoft.Attestation` * `Microsoft.Compute` * `Microsoft.Insights` * `Microsoft.ManagedIdentity` @@ -121,6 +197,7 @@ The following [resource providers need to be registered](https://learn.microsoft By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + * `*/register/action` \[1] * `Microsoft.Authorization/roleAssignments/*` * `Microsoft.Authorization/roleDefinitions/*` @@ -130,7 +207,8 @@ To [create the IAM configuration](../workflows/config.md#creating-an-iam-configu The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] + +* `Microsoft.Attestation/attestationProviders/*` * `Microsoft.Compute/virtualMachineScaleSets/*` * `Microsoft.Insights/components/*` * `Microsoft.ManagedIdentity/userAssignedIdentities/*` @@ -148,15 +226,17 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.roles.create` +* `iam.roles.delete` +* `iam.roles.get` * `iam.serviceAccountKeys.create` * `iam.serviceAccountKeys.delete` * `iam.serviceAccountKeys.get` @@ -169,6 +249,7 @@ To [create the IAM configuration](../workflows/config.md#creating-an-iam-configu Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + * `compute.addresses.createInternal` * `compute.addresses.deleteInternal` * `compute.addresses.get` @@ -182,6 +263,11 @@ To [create a Constellation cluster](../workflows/create.md), you need the follow * `compute.firewalls.delete` * `compute.firewalls.get` * `compute.firewalls.update` +* `compute.forwardingRules.create` +* `compute.forwardingRules.delete` +* `compute.forwardingRules.get` +* `compute.forwardingRules.setLabels` +* `compute.forwardingRules.list` * `compute.globalAddresses.create` * `compute.globalAddresses.delete` * `compute.globalAddresses.get` @@ -234,60 +320,16 @@ Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and ` Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + -To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. -To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ec2:DescribeAccountAttributes", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreatePolicy", - "iam:CreateRole", - "iam:DeleteInstanceProfile", - "iam:DeletePolicy", - "iam:DeletePolicyVersion", - "iam:DeleteRole", - "iam:DetachRolePolicy", - "iam:GetInstanceProfile", - "iam:GetPolicy", - "iam:GetPolicyVersion", - "iam:GetRole", - "iam:ListAttachedRolePolicies", - "iam:ListInstanceProfilesForRole", - "iam:ListPolicyVersions", - "iam:ListRolePolicies", - "iam:PassRole", - "iam:RemoveRoleFromInstanceProfile", - "sts:GetCallerIdentity" - ], - "Resource": "*" - } - ] -} -``` - -The built-in `AdministratorAccess` policy is a superset of these permissions. - -To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). - - -The built-in `PowerUserAccess` policy is a superset of these permissions. - -Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - - + + ### Authentication @@ -297,49 +339,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - - -**Testing** - -Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). - -**Production** - -Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: - -```bash -az login -``` - -Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - - - -**Testing** - -You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. - -**Production** - -Use one of the following options on a trusted machine: - -- Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) - - ```bash - gcloud auth application-default login - ``` - - This will ask you to log-in to your Google account and create your credentials. - The Constellation CLI will automatically load these credentials when needed. - -- Set up a service account and pass the credentials manually - - Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - - + + **Testing** @@ -355,10 +356,91 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file with the credentials from the User Access Token under: + * Linux: `~/.config/openstack/clouds.yaml` + * macOS: `/Users//Library/Application Support/openstack/clouds.yaml` or `/etc/openstack/clouds.yaml` + * Windows: `%AppData%\openstack\clouds.yaml` - + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_OPENSTACK_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +:::caution + +`project_id` refers to the ID of your OpenStack project. The STACKIT portal also shows the STACKIT ID that's associated with your project in some places. Make sure you insert the OpenStack project ID in the `clouds.yaml` file. + +::: + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + ## Next steps diff --git a/docs/docs/getting-started/marketplaces.md b/docs/docs/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/docs/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/docs/overview/clouds.md b/docs/docs/overview/clouds.md index 3ccbb0d6d..b2695d28e 100644 --- a/docs/docs/overview/clouds.md +++ b/docs/docs/overview/clouds.md @@ -5,50 +5,58 @@ What works on which cloud? Currently, Confidential VMs (CVMs) are available in v For Constellation, the ideal environment provides the following: 1. Ability to run arbitrary software and images inside CVMs -2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or, in the future, Intel TDX (available in Xeon CPUs from the Sapphire Rapids generation onward) +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) 3. Ability for CVM guests to obtain raw hardware attestation statements 4. Reviewable, open-source firmware inside CVMs 5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) (1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. -The following table summarizes the state of features for different infrastructures as of June 2023. +The following table summarizes the state of features for different infrastructures. -| **Feature** | **Azure** | **GCP** | **AWS** | **OpenStack (Yoga)** | -|-----------------------------------|-----------|---------|---------|----------------------| -| **1. Custom images** | Yes | Yes | Yes | Yes | -| **2. SEV-SNP or TDX** | Yes | Yes | Yes | Depends on kernel/HV | -| **3. Raw guest attestation** | Yes | Yes | Yes | Depends on kernel/HV | -| **4. Reviewable firmware** | No* | No | Yes | Depends on kernel/HV | -| **5. Confidential measured boot** | Yes | No | No | Depends on kernel/HV | - -## Microsoft Azure - -With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. -Regarding (3), Azure provides direct access to remote-attestation statements. -The CVM firmware running in VM Privilege Level (VMPL) 0 provides a vTPM (5), but it's closed source (4). -This firmware is signed by Azure. -The signature is reflected in the remote-attestation statements of CVMs. -Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). - -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. - -## Google Cloud Platform (GCP) - -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. -CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. -However, regarding (4), the CVMs still include closed-source firmware. - -Intel and Google have [collaborated](https://cloud.google.com/blog/products/identity-security/rsa-google-intel-confidential-computing-more-secure) to enhance the security of TDX, and have recently [revealed](https://venturebeat.com/security/intel-launches-confidential-computing-solution-for-virtual-machines/) their plans to make TDX compatible with Google Cloud. +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | ## Amazon Web Services (AWS) Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). -Regarding (3), AWS provides direct access to remote-attestation statements. +Regarding (3), AWS provides direct access to attestation statements. However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + ## OpenStack OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. diff --git a/docs/docs/overview/confidential-kubernetes.md b/docs/docs/overview/confidential-kubernetes.md index ca20df4de..bff8c3322 100644 --- a/docs/docs/overview/confidential-kubernetes.md +++ b/docs/docs/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/docs/overview/license.md b/docs/docs/overview/license.md index caec5aeaa..98a9cbf94 100644 --- a/docs/docs/overview/license.md +++ b/docs/docs/overview/license.md @@ -1,27 +1,15 @@ # License -## Source code +Constellation is available under the [Business Source License 1.1](https://github.com/edgelesssys/constellation/blob/main/LICENSE). -Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). +You may use it free of charge for non-production use ("Community License"). -## Binaries +## Enterprise License -Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). - -These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. - -The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. - -### Community License - -You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. - -### Enterprise License - -Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). +Enterprise Licenses permit production use and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. -### Azure Marketplace +## CSP Marketplaces -Constellation is available through the Azure Marketplace. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your Azure account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/docs/overview/performance/compute.md b/docs/docs/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/docs/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/docs/overview/performance/io.md b/docs/docs/overview/performance/io.md index dc7cf3d8b..3ae796f8a 100644 --- a/docs/docs/overview/performance/io.md +++ b/docs/docs/overview/performance/io.md @@ -58,7 +58,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. diff --git a/docs/docs/overview/performance/performance.md b/docs/docs/overview/performance/performance.md index 7f22a693e..59bf86602 100644 --- a/docs/docs/overview/performance/performance.md +++ b/docs/docs/overview/performance/performance.md @@ -1,18 +1,10 @@ # Performance analysis of Constellation -This section provides a comprehensive examination of the performance characteristics of Constellation, encompassing various aspects, including runtime encryption, I/O benchmarks, and real-world applications. +This section provides a comprehensive examination of the performance characteristics of Constellation. -## Impact of runtime encryption on performance +## Runtime encryption -All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. - -### AMD and Azure benchmarking - -AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. - -### AMD and Google benchmarking - -Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. ## I/O performance benchmarks diff --git a/docs/docs/overview/product.md b/docs/docs/overview/product.md index ba7181aa9..4b5d90706 100644 --- a/docs/docs/overview/product.md +++ b/docs/docs/overview/product.md @@ -6,6 +6,7 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/docs/reference/cli.md b/docs/docs/reference/cli.md index 6a3e7c429..c5d7c652b 100644 --- a/docs/docs/reference/cli.md +++ b/docs/docs/reference/cli.md @@ -39,6 +39,7 @@ Commands: * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile * [version](#constellation-version): Display version of this CLI * [init](#constellation-init): Initialize the Constellation cluster +* [ssh](#constellation-ssh): Generate a certificate for emergency SSH access ## constellation config @@ -78,9 +79,10 @@ constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] ### Options ``` - -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-trustedlaunch|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used -h, --help help for generate - -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.27") + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.31") + -t, --tags strings additional tags for created resources given a list of key=value ``` ### Options inherited from parent commands @@ -654,6 +656,7 @@ constellation iam create azure [flags] --region string region the resources will be created in, e.g., westus (required) --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set ``` ### Options inherited from parent commands @@ -682,13 +685,13 @@ constellation iam create gcp [flags] ### Options ``` - -h, --help help for gcp - --projectID string ID of the GCP project the configuration will be created in (required) - Find it on the welcome screen of your project: https://console.cloud.google.com/welcome - --serviceAccountID string ID for the service account that will be created (required) - Must be 6 to 30 lowercase letters, digits, or hyphens. - --zone string GCP zone the cluster will be deployed in (required) - Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available + -h, --help help for gcp + --prefix string Prefix for the service account ID and VM ID that will be created (required) + Must be letters, digits, or hyphens. + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available ``` ### Options inherited from parent commands @@ -840,3 +843,31 @@ constellation init [flags] -C, --workspace string path to the Constellation workspace ``` +## constellation ssh + +Generate a certificate for emergency SSH access + +### Synopsis + +Generate a certificate for emergency SSH access to your SSH-enabled constellation cluster. + +``` +constellation ssh [flags] +``` + +### Options + +``` + -h, --help help for ssh + --key string the path to an existing SSH public key +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/docs/reference/migration.md b/docs/docs/reference/migration.md index 36680eef6..eb55d650b 100644 --- a/docs/docs/reference/migration.md +++ b/docs/docs/reference/migration.md @@ -3,39 +3,93 @@ This document describes breaking changes and migrations between Constellation releases. Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. -## Migrating from Azure's service principal authentication to managed identity authentication +## Migrations to v2.23.0 -- The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. -- To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. -- Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. -- To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: +### GCP + +GCP will require the additional permission `compute.forwardingRules.list`. Please update your IAM roles using `constellation iam upgrade apply`. + +## Migrations to v2.19.1 + +### Azure + +* During the upgrade, security rules are migrated and the old ones need to be cleaned up manually by the user. The below script shows how to delete them through the Azure CLI: + +```bash +#!/usr/bin/env bash +name="" # the name provided in the config +uid="" # the cluster id can be retrieved via `yq '.infrastructure.uid' constellation-state.yaml` +resource_group="" # the RG can be retrieved via `yq '.provider.azure.resourceGroup' constellation-conf.yaml` + +rules=( + "kubernetes" + "bootstrapper" + "verify" + "recovery" + "join" + "debugd" + "konnectivity" +) + +for rule in "${rules[@]}"; do + echo "Deleting rule: ${rule}" + az network nsg rule delete \ + --resource-group "${resource_group}" \ + --nsg-name "${name}-${uid}" \ + --name "${rule}" +done + +echo "All specified rules have been deleted." +``` + +## Migrating from CLI versions before 2.21.1 + +### AWS + +* AWS clusters that use `LoadBalancer` resources require more IAM permissions. Please upgrade your IAM roles using `constellation iam upgrade apply`. This will show necessary changes and apply them, if desired. + +## Migrating from CLI versions before 2.19.0 + +### Azure + +* To allow seamless upgrades on Azure when Kubernetes services of type `LoadBalancer` are deployed, the target + load balancer in which the `cloud-controller-manager` creates load balancing rules was changed. Instead of using the load balancer + created and maintained by the CLI's Terraform code, the `cloud-controller-manager` now creates its own load balancer in Azure. + If your Constellation has services of type `LoadBalancer`, please remove them before the upgrade and re-apply them + afterward. + +## Migrating from CLI versions before 2.18.0 + +* The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +* To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +* Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +* To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. - ## Migrating from CLI versions before 2.10 -- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. -- The global `nodeGroups` field was added. -- The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. -- The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. +* AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +* The global `nodeGroups` field was added. +* The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +* The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. ## Migrating from CLI versions before 2.9 -- The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication +* The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication ## Migrating from CLI versions before 2.8 -- The `measurements` field for each cloud service provider was replaced with a global `attestation` field. -- The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. -- The optional global field `attestationVariant` was replaced by the now required `attestation` field. +* The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +* The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +* The optional global field `attestationVariant` was replaced by the now required `attestation` field. ## Migrating from CLI versions before 2.3 -- The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. +* The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). -- The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +* The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration:

Show all @@ -65,10 +119,11 @@ Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to a | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | +
-- The `enforcedMeasurements` field has been removed and merged with the `measurements` field. - - To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` - - To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: +* The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + * To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + * To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: ```diff measurements: diff --git a/docs/docs/workflows/config.md b/docs/docs/workflows/config.md index 165100b81..7868ff1be 100644 --- a/docs/docs/workflows/config.md +++ b/docs/docs/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,52 +14,47 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - - -```bash -constellation config generate azure -``` - - - - -```bash -constellation config generate gcp -``` - - - + + ```bash constellation config generate aws ``` - - + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - - -By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. - -You can also run `constellation config instance-types` to get the list of all supported options. - - - - -By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. -Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. @@ -67,10 +62,41 @@ If you are using the attestation variant `awsNitroTPM`, you can choose any of th The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. - - + + -Fill the desired VM type into the **instanceType** fields in the `constellation-conf.yml` file. +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. ## Creating additional node groups @@ -109,11 +135,13 @@ This configuration creates an additional node group `high_cpu` with a larger ins You can use the field `zone` to specify what availability zone nodes of the group are placed in. On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. Consult the documentation of your cloud provider for more information: * [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) * [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) * [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) ## Choosing a Kubernetes version @@ -125,44 +153,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. - - - -You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). - -```bash -constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest -``` - -This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. - -Note that CVMs are currently only supported in a few regions, check [Azure's products available by region](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). These are: - -* `westus` -* `eastus` -* `northeurope` -* `westeurope` -* `southeastasia` - -Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - - - -You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). - -```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test -``` - -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. - -Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. - -Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -186,73 +178,62 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --prefix=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - - -* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. - - You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). - -* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. - - You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). - -* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. CVMs are currently only supported in a few regions, check [Azure's products available by region](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). These are: - - * `westus` - * `eastus` - * `northeurope` - * `westeurope` - * `southeastasia` - -* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. - -* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. - - Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. - - Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. - - The user-assigned identity is used by instances of the cluster to access other cloud resources. - For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). - - - - - -* **project**: The ID of your GCP project, e.g., `constellation-129857`. - - You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). - -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. - - You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). - -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. - - You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). - -* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: - - * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` - * `Compute Network Admin (roles/compute.networkAdmin)` - * `Compute Security Admin (roles/compute.securityAdmin)` - * `Compute Storage Admin (roles/compute.storageAdmin)` - * `Service Account User (roles/iam.serviceAccountUser)` - - Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - - - - + + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -283,9 +264,75 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + + - +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/docs/workflows/create.md b/docs/docs/workflows/create.md index 54bc9dcbc..6074ebb16 100644 --- a/docs/docs/workflows/create.md +++ b/docs/docs/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -27,8 +27,8 @@ If you don't have a cloud subscription, you can also set up a [local Constellati Before you create the cluster, make sure to have a [valid configuration file](./config.md). - - + + ```bash constellation apply @@ -36,8 +36,8 @@ constellation apply `apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. This provides flexibility in DevOps and can meet potential regulatory requirements. @@ -56,7 +56,7 @@ management tooling of your choice. You need to keep the essential functionality :::info - On Azure, if the enforcement policy is set to `MAAFallback` in `constellation-config.yaml`, a manual update to the MAA provider's policy is necessary. + On Azure, a manual update to the MAA provider's policy is necessary. You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). ```bash @@ -77,8 +77,8 @@ With the required cloud resources set up, continue with initializing your cluste constellation apply --skip-phases=infrastructure ``` - - + + Finally, configure `kubectl` for your cluster: diff --git a/docs/docs/workflows/lb.md b/docs/docs/workflows/lb.md index 11e403237..868e61076 100644 --- a/docs/docs/workflows/lb.md +++ b/docs/docs/workflows/lb.md @@ -4,12 +4,25 @@ Constellation integrates the native load balancers of each CSP. Therefore, to ex ## Internet-facing LB service on AWS -To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancing Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. -Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/). +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). :::caution Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. ::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/docs/workflows/recovery.md b/docs/docs/workflows/recovery.md index 955981749..592ae247b 100644 --- a/docs/docs/workflows/recovery.md +++ b/docs/docs/workflows/recovery.md @@ -13,11 +13,42 @@ The first step to recovery is identifying when a cluster becomes unhealthy. Usually, this can be first observed when the Kubernetes API server becomes unresponsive. You can check the health status of the nodes via the cloud service provider (CSP). -Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). +Constellation provides logging information on the boot process and status via serial console output. In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +82,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,12 +118,12 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + -First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. -Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. In the serial console output, search for `Waiting for decryption key`. Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): @@ -118,8 +149,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster @@ -131,7 +162,7 @@ Recovering a cluster requires the following parameters: A cluster can be recovered like this: ```bash -$ constellation recover --master-secret constellation-mastersecret.json +$ constellation recover Pushed recovery key. Pushed recovery key. Pushed recovery key. diff --git a/docs/docs/workflows/reproducible-builds.md b/docs/docs/workflows/reproducible-builds.md new file mode 100644 index 000000000..e3bc46095 --- /dev/null +++ b/docs/docs/workflows/reproducible-builds.md @@ -0,0 +1,63 @@ +# Reproduce released artifacts + +Constellation has first-class support for [reproducible builds](https://reproducible-builds.org). +Reproducing the released artifacts is an alternative to [signature verification](verify-cli.md) that doesn't require trusting Edgeless Systems' release process. +The following sections describe how to rebuild an artifact and how Constellation ensures that the rebuild reproduces the artifacts bit-by-bit. + +## Build environment prerequisites + +The build systems used by Constellation - [Bazel](https://bazel.build/) and [Nix](https://nixos.org) - are designed for deterministic, reproducible builds. +These two dependencies should be the only prerequisites for a successful build. +However, it can't be ruled out completely that peculiarities of the host affect the build result. +Thus, we recommend the following host setup for best results: + +1. A Linux operating system not older than v5.4. +2. The GNU C library not older than v2.31 (avoid `musl`). +3. GNU `coreutils` not older than v8.30 (avoid `busybox`). +4. An `ext4` filesystem for building. +5. AppArmor turned off. + +This is given, for example, on an Ubuntu 22.04 system, which is also used for reproducibility tests. + +:::note + +To avoid any backwards-compatibility issues, the host software versions should also not be much newer than the Constellation release. + +::: + +## Run the build + +The following instructions outline qualitatively how to reproduce a build. +Constellation implements these instructions in the [Reproducible Builds workflow](https://github.com/edgelesssys/constellation/actions/workflows/reproducible-builds.yml), which continuously tests for reproducibility. +The workflow is a good place to look up specific version numbers and build steps. + +1. Check out the Constellation repository at the tag corresponding to the release. + + ```bash + git clone https://github.com/edgelesssys/constellation.git + cd constellation + git checkout v2.20.0 + ``` + +2. [Install the Bazel release](https://bazel.build/install) specified in `.bazelversion`. +3. [Install Nix](https://nixos.org/download/) (any recent version should do). +4. Run the build with `bazel build $target` for one of the following targets of interest: + + ```data + //cli:cli_enterprise_darwin_amd64 + //cli:cli_enterprise_darwin_arm64 + //cli:cli_enterprise_linux_amd64 + //cli:cli_enterprise_linux_arm64 + //cli:cli_enterprise_windows_amd64 + ``` + +5. Compare the build result with the downloaded release artifact. + + + +## Feedback + +Reproduction failures often indicate a bug in the build system or in the build definitions. +Therefore, we're interested in any reproducibility issues you might encounter. +[Start a bug report](https://github.com/edgelesssys/constellation/issues/new/choose) and describe the details of your build environment. +Make sure to include your result binary or a [`diffoscope`](https://diffoscope.org/) report, if possible. diff --git a/docs/docs/workflows/sbom.md b/docs/docs/workflows/sbom.md index 9ef6eb65c..6c1702dee 100644 --- a/docs/docs/workflows/sbom.md +++ b/docs/docs/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -11,13 +11,15 @@ SBOMs for Constellation are generated using [Syft](https://github.com/anchore/sy :::note The public key for Edgeless Systems' long-term code-signing key is: + ``` -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at https://edgeless.systems/es.pub and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -38,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/docs/workflows/scale.md b/docs/docs/workflows/scale.md index 06898ad0c..28f19e3f1 100644 --- a/docs/docs/workflows/scale.md +++ b/docs/docs/workflows/scale.md @@ -51,30 +51,36 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + -1. Go to Auto Scaling Groups and select the worker ASG to scale up. -2. Click **Edit** -3. Set the new (increased) **Desired capacity** and **Update**. +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. - - + + ## Control-plane node scaling @@ -82,30 +88,35 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + + - +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + -1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. -2. Click **Edit** -3. Set the new (increased) **Desired capacity** and **Update**. +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/docs/workflows/storage.md b/docs/docs/workflows/storage.md index 9e3d96346..a5c52be90 100644 --- a/docs/docs/workflows/storage.md +++ b/docs/docs/workflows/storage.md @@ -9,11 +9,11 @@ Cloud service providers (CSPs) offer their own CSI-based solutions for cloud sto ## Confidential storage Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). -Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, and GCP. +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. However, their encryption takes place in the storage backend and is managed by the CSP. Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. -To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, and GCE PD, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). @@ -21,30 +21,37 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. - - + + -**Constellation CSI driver for AWS Elastic Block Store** -Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. -Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -53,66 +60,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - - -Azure comes with two storage classes by default. - -* `encrypted-rwo` - * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) - * ext-4 filesystem - * Encryption of all data written to disk -* `integrity-encrypted-rwo` - * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) - * ext-4 filesystem - * Encryption of all data written to disk - * Integrity protection of data written to disk - -For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). - -:::info - -The default storage class is set to `encrypted-rwo` for performance reasons. -If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. - -Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. -Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. - -Note that volume expansion isn't supported for integrity-protected disks. - -::: - - - - -GCP comes with two storage classes by default. - -* `encrypted-rwo` - * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) - * ext-4 filesystem - * Encryption of all data written to disk -* `integrity-encrypted-rwo` - * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) - * ext-4 filesystem - * Encryption of all data written to disk - * Integrity protection of data written to disk - -For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). - -:::info - -The default storage class is set to `encrypted-rwo` for performance reasons. -If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. - -Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. -Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. - -Note that volume expansion isn't supported for integrity-protected disks. - -::: - - - + + AWS comes with two storage classes by default. @@ -140,8 +89,95 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/docs/docs/workflows/terminate.md b/docs/docs/workflows/terminate.md index 58c274bdd..2c45bebe3 100644 --- a/docs/docs/workflows/terminate.md +++ b/docs/docs/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-state.yaml constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/docs/workflows/terraform-provider.md b/docs/docs/workflows/terraform-provider.md index ec9b222e4..c7a795d3f 100644 --- a/docs/docs/workflows/terraform-provider.md +++ b/docs/docs/workflows/terraform-provider.md @@ -1,7 +1,6 @@ # Use the Terraform provider The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. - The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. ## Prerequisites @@ -22,22 +21,32 @@ This example shows how to set up a Constellation cluster with the reference IAM 2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. 3. Initialize and apply the Terraform configuration. - - - When creating a cluster on Azure, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you - can also do it manually. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +:::info +On SEV-SNP, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you can also do it manually. ```bash terraform init terraform apply -target module.azure_iam # adjust resource path if not using the example configuration terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration - constellation maa-patch $(terraform output maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration ``` - Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. - - Use the following policy if manually performing the patch. + Use the following policy if manually performing the patch. ``` version= 1.0; @@ -56,8 +65,21 @@ This example shows how to set up a Constellation cluster with the reference IAM { }; ``` - - + +::: + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + + Initialize the providers and apply the configuration. ```bash @@ -66,8 +88,8 @@ This example shows how to set up a Constellation cluster with the reference IAM ``` Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. - - + + Initialize the providers and apply the configuration. ```bash @@ -76,8 +98,9 @@ This example shows how to set up a Constellation cluster with the reference IAM ``` Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. - - + + + 4. Connect to the cluster. ```bash diff --git a/docs/docs/workflows/troubleshooting.md b/docs/docs/workflows/troubleshooting.md index 633053e0b..7ed26ae7f 100644 --- a/docs/docs/workflows/troubleshooting.md +++ b/docs/docs/workflows/troubleshooting.md @@ -40,6 +40,24 @@ Or alternatively, for `terminate`: ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate ``` +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + ### Nodes fail to join with error `untrusted measurement value` This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). @@ -95,49 +113,14 @@ check if the encountered [issue is known](https://github.com/edgelesssys/constel ## Diagnosing issues -### Cloud logging +### Logs -To provide information during early stages of a node's boot process, Constellation logs messages to the log systems of the cloud providers. Since these offerings **aren't** confidential, only generic information without any sensitive values is stored. This provides administrators with a high-level understanding of the current state of a node. +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). -You can view this information in the following places: +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. - - - -1. In your Azure subscription find the Constellation resource group. -2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. -3. On the left-hand side go to `Logs`, which is located in the section `Monitoring`. - - Close the Queries page if it pops up. -5. In the query text field type in `traces`, and click `Run`. - -To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - - - -1. Select the project that hosts Constellation. -2. Go to the `Compute Engine` service. -3. On the right-hand side of a VM entry select `More Actions` (a stacked ellipsis) - - Select `View logs` - -To **find the disk UUIDs** use the following query: `resource.type="gce_instance" text_payload=~"Disk UUID:.*\n" logName=~".*/constellation-boot-log"` - -:::info - -Constellation uses the default bucket to store logs. Its [default retention period is 30 days](https://cloud.google.com/logging/quotas#logs_retention_periods). - -::: - - - - -1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) -2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) -3. Select the log group that matches the name of your cluster. -4. Select the log stream for control or worker type nodes. - - - +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). ### Node shell access @@ -166,3 +149,54 @@ Debugging via a shell on a node is [directly supported by Kubernetes](https://ku ```bash kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj ``` + +### Emergency SSH access + +Emergency SSH access to nodes can be useful to diagnose issues or download important data even if the Kubernetes API isn't reachable anymore. + +1. Enter the `constellation-terraform` directory in your Constellation workspace and enable emergency SSH access to the cluster: + + ```bash + cd constellation-terraform + echo "emergency_ssh = true" >> ./terraform.tfvars + terraform apply + ``` + +2. Sign an existing SSH key with your master secret: + + ```bash + cd ../ # go back to your Constellation workspace + constellation ssh --key your_public_key.pub + ``` + + A certificate is written to `constellation_cert.pub`. + + The certificate is valid for 24 hours and enables you to access your Constellation nodes using + [certificate based authentication](https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Certificate-based_Authentication). + +3. Now you can connect to any Constellation node using your certificate and your private key. + + ```bash + ssh -o CertificateFile=constellation_cert.pub -o UserKnownHostsFile=./known_hosts -i root@ + ``` + + Normally, you don't have access to the Constellation nodes since they reside in a private network. + To access those nodes anyways, you can use your Constellation load balancer as a proxy jump host. + For this, use something along the following SSH client configuration: + + ```text + Host + ProxyJump none + + Host * + IdentityFile + PreferredAuthentications publickey + CertificateFile=constellation_cert.pub + UserKnownHostsFile=./known_hosts + User root + ProxyJump + ``` + + With this configuration you can connect to a Constellation node using `ssh -F `. + You can obtain the private node IP and the public IP of the load balancer using your CSP's web UI. Note that if + you use the load balancers domain name, ssh host certificate verification doesn't work, so using the public IP is recommended. diff --git a/docs/docs/workflows/trusted-launch.md b/docs/docs/workflows/trusted-launch.md index 9bc7e785f..d6d01d8eb 100644 --- a/docs/docs/workflows/trusted-launch.md +++ b/docs/docs/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: diff --git a/docs/docs/workflows/upgrade.md b/docs/docs/workflows/upgrade.md index 7348c0dbc..3db2ecad6 100644 --- a/docs/docs/workflows/upgrade.md +++ b/docs/docs/workflows/upgrade.md @@ -1,6 +1,6 @@ # Upgrade your cluster -Constellation provides an easy way to upgrade all components of your cluster, without disrupting it's availability. +Constellation provides an easy way to upgrade all components of your cluster, without disrupting its availability. Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. To learn about available versions you use the `upgrade check` command. diff --git a/docs/docs/workflows/verify-cli.md b/docs/docs/workflows/verify-cli.md index 1280c51b0..e33569d37 100644 --- a/docs/docs/workflows/verify-cli.md +++ b/docs/docs/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,11 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/docs/workflows/verify-cluster.md b/docs/docs/workflows/verify-cluster.md index 20d416790..b6595ebf2 100644 --- a/docs/docs/workflows/verify-cluster.md +++ b/docs/docs/workflows/verify-cluster.md @@ -88,6 +88,7 @@ The `verify` command also allows you to verify any Constellation deployment that * The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. * The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. For example: diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 2dc60e883..d5d220fdf 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -13,6 +13,7 @@ async function createConfig() { baseUrl: '/constellation/', onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'throw', + onBrokenAnchors: 'throw', favicon: 'img/favicon.ico', // GitHub pages deployment config. @@ -22,9 +23,11 @@ async function createConfig() { // scripts scripts: [ - { src: 'https://plausible.io/js/plausible.js', async: true, defer: true, 'data-domain': 'docs.edgeless.systems' }, - { id: "Cookiebot", src: "https://consent.cookiebot.com/uc.js", "data-cbid": "a0cc864f-0b67-49be-8d65-9ed354de2ee6", "data-blockingmode": "auto" }, - { id: "CookieDeclaration", src: "https://consent.cookiebot.com/a0cc864f-0b67-49be-8d65-9ed354de2ee6/cd.js" } + { + src: '/constellation/gtagman.js', + async: true, + "data-cookieconsent": "ignore", + }, ], // Even if you don't use internalization, you can use this field to set useful @@ -58,10 +61,6 @@ async function createConfig() { theme: { customCss: require.resolve('./src/css/custom.css'), }, - gtag: { - trackingID: 'G-3DVYB2CHLG', - anonymizeIP: true, - } }), ], ], diff --git a/docs/package-lock.json b/docs/package-lock.json new file mode 100644 index 000000000..cb381bcac --- /dev/null +++ b/docs/package-lock.json @@ -0,0 +1,19420 @@ +{ + "name": "constellation-docs", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "constellation-docs", + "version": "2.0.0", + "dependencies": { + "@cmfcmf/docusaurus-search-local": "1.2.0", + "@docusaurus/core": "3.8.0", + "@docusaurus/preset-classic": "3.8.0", + "@docusaurus/theme-mermaid": "3.8.0", + "@mdx-js/react": "3.1.0", + "asciinema-player": "3.10.0", + "clsx": "2.1.1", + "prism-react-renderer": "2.4.1", + "react": "18.3.1", + "react-dom": "18.3.1" + }, + "devDependencies": { + "@docusaurus/module-type-aliases": "3.8.0", + "@docusaurus/types": "3.8.0" + }, + "engines": { + "node": ">=16.14" + } + }, + "node_modules/@algolia/autocomplete-core": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.17.7.tgz", + "integrity": "sha512-BjiPOW6ks90UKl7TwMv7oNQMnzU+t/wk9mgIDi6b1tXpUek7MW0lbNOUHpvam9pe3lVCf4xPFT+lK7s+e+fs7Q==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-plugin-algolia-insights": "1.17.7", + "@algolia/autocomplete-shared": "1.17.7" + } + }, + "node_modules/@algolia/autocomplete-js": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-js/-/autocomplete-js-1.17.7.tgz", + "integrity": "sha512-4rCCg2B5x6GYzLfDZ3QipWydznbaMjoIwNSEbjpJ9cd/0+4nDpRWuBPxgOSsGmE4BFEor2iwQw4uCY6RrBdpjA==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-core": "1.17.7", + "@algolia/autocomplete-preset-algolia": "1.17.7", + "@algolia/autocomplete-shared": "1.17.7", + "htm": "^3.1.1", + "preact": "^10.13.2" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.5.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-plugin-algolia-insights": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-plugin-algolia-insights/-/autocomplete-plugin-algolia-insights-1.17.7.tgz", + "integrity": "sha512-Jca5Ude6yUOuyzjnz57og7Et3aXjbwCSDf/8onLHSQgw1qW3ALl9mrMWaXb5FmPVkV3EtkD2F/+NkT6VHyPu9A==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-shared": "1.17.7" + }, + "peerDependencies": { + "search-insights": ">= 1 < 3" + } + }, + "node_modules/@algolia/autocomplete-preset-algolia": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.17.7.tgz", + "integrity": "sha512-ggOQ950+nwbWROq2MOCIL71RE0DdQZsceqrg32UqnhDz8FlO9rL8ONHNsI2R1MH0tkgVIDKI/D0sMiUchsFdWA==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-shared": "1.17.7" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-shared": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.17.7.tgz", + "integrity": "sha512-o/1Vurr42U/qskRSuhBH+VKxMvkkUVTLU6WZQr+L5lGZZLYWyhdzWjW0iGXY7EkwRTjBqvN2EsR81yCTGV/kmg==", + "license": "MIT", + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-theme-classic": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-theme-classic/-/autocomplete-theme-classic-1.17.7.tgz", + "integrity": "sha512-8sxnzRCPxyKZJxbG7EUpV/3AssQOjn+Zq/nvzks+BwbkAcpiLRBsXjvlIIsV4l36bZ+/Ri++ttAflGDPrRfn1A==", + "license": "MIT" + }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.24.0.tgz", + "integrity": "sha512-t63W9BnoXVrGy9iYHBgObNXqYXM3tYXCjDSHeNwnsc324r4o5UiVKUiAB4THQ5z9U5hTj6qUvwg/Ez43ZD85ww==", + "license": "MIT", + "dependencies": { + "@algolia/cache-common": "4.24.0" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.24.0.tgz", + "integrity": "sha512-emi+v+DmVLpMGhp0V9q9h5CdkURsNmFC+cOS6uK9ndeJm9J4TiqSvPYVu+THUP8P/S08rxf5x2P+p3CfID0Y4g==", + "license": "MIT" + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.24.0.tgz", + "integrity": "sha512-gDrt2so19jW26jY3/MkFg5mEypFIPbPoXsQGQWAi6TrCPsNOSEYepBMPlucqWigsmEy/prp5ug2jy/N3PVG/8w==", + "license": "MIT", + "dependencies": { + "@algolia/cache-common": "4.24.0" + } + }, + "node_modules/@algolia/client-abtesting": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-abtesting/-/client-abtesting-5.19.0.tgz", + "integrity": "sha512-dMHwy2+nBL0SnIsC1iHvkBao64h4z+roGelOz11cxrDBrAdASxLxmfVMop8gmodQ2yZSacX0Rzevtxa+9SqxCw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-abtesting/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-abtesting/node_modules/@algolia/requester-browser-xhr": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.19.0.tgz", + "integrity": "sha512-GfnhnQBT23mW/VMNs7m1qyEyZzhZz093aY2x8p0era96MMyNv8+FxGek5pjVX0b57tmSCZPf4EqNCpkGcGsmbw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-abtesting/node_modules/@algolia/requester-node-http": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.19.0.tgz", + "integrity": "sha512-p6t8ue0XZNjcRiqNkb5QAM0qQRAKsCiebZ6n9JjWA+p8fWf8BvnhO55y2fO28g3GW0Imj7PrAuyBuxq8aDVQwQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.24.0.tgz", + "integrity": "sha512-adcvyJ3KjPZFDybxlqnf+5KgxJtBjwTPTeyG2aOyoJvx0Y8dUQAEOEVOJ/GBxX0WWNbmaSrhDURMhc+QeevDsA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "4.24.0", + "@algolia/client-search": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.24.0.tgz", + "integrity": "sha512-y8jOZt1OjwWU4N2qr8G4AxXAzaa8DBvyHTWlHzX/7Me1LX8OayfgHexqrsL4vSBcoMmVw2XnVW9MhL+Y2ZDJXg==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "4.24.0", + "@algolia/client-search": "4.24.0", + "@algolia/requester-common": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.24.0.tgz", + "integrity": "sha512-bc2ROsNL6w6rqpl5jj/UywlIYC21TwSSoFHKl01lYirGMW+9Eek6r02Tocg4gZ8HAw3iBvu6XQiM3BEbmEMoiA==", + "license": "MIT", + "dependencies": { + "@algolia/requester-common": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/@algolia/client-insights": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-insights/-/client-insights-5.19.0.tgz", + "integrity": "sha512-xPOiGjo6I9mfjdJO7Y+p035aWePcbsItizIp+qVyfkfZiGgD+TbNxM12g7QhFAHIkx/mlYaocxPY/TmwPzTe+A==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-insights/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-insights/node_modules/@algolia/requester-browser-xhr": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.19.0.tgz", + "integrity": "sha512-GfnhnQBT23mW/VMNs7m1qyEyZzhZz093aY2x8p0era96MMyNv8+FxGek5pjVX0b57tmSCZPf4EqNCpkGcGsmbw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-insights/node_modules/@algolia/requester-node-http": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.19.0.tgz", + "integrity": "sha512-p6t8ue0XZNjcRiqNkb5QAM0qQRAKsCiebZ6n9JjWA+p8fWf8BvnhO55y2fO28g3GW0Imj7PrAuyBuxq8aDVQwQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.24.0.tgz", + "integrity": "sha512-l5FRFm/yngztweU0HdUzz1rC4yoWCFo3IF+dVIVTfEPg906eZg5BOd1k0K6rZx5JzyyoP4LdmOikfkfGsKVE9w==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "4.24.0", + "@algolia/requester-common": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/@algolia/client-query-suggestions": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-query-suggestions/-/client-query-suggestions-5.19.0.tgz", + "integrity": "sha512-6fcP8d4S8XRDtVogrDvmSM6g5g6DndLc0pEm1GCKe9/ZkAzCmM3ZmW1wFYYPxdjMeifWy1vVEDMJK7sbE4W7MA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-query-suggestions/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-query-suggestions/node_modules/@algolia/requester-browser-xhr": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.19.0.tgz", + "integrity": "sha512-GfnhnQBT23mW/VMNs7m1qyEyZzhZz093aY2x8p0era96MMyNv8+FxGek5pjVX0b57tmSCZPf4EqNCpkGcGsmbw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-query-suggestions/node_modules/@algolia/requester-node-http": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.19.0.tgz", + "integrity": "sha512-p6t8ue0XZNjcRiqNkb5QAM0qQRAKsCiebZ6n9JjWA+p8fWf8BvnhO55y2fO28g3GW0Imj7PrAuyBuxq8aDVQwQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.24.0.tgz", + "integrity": "sha512-uRW6EpNapmLAD0mW47OXqTP8eiIx5F6qN9/x/7HHO6owL3N1IXqydGwW5nhDFBrV+ldouro2W1VX3XlcUXEFCA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "4.24.0", + "@algolia/requester-common": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/@algolia/events": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@algolia/events/-/events-4.0.1.tgz", + "integrity": "sha512-FQzvOCgoFXAbf5Y6mYozw2aj5KCJoA3m4heImceldzPSMbdyS4atVjJzXKMsfX3wnZTFYwkkt8/z8UesLHlSBQ==", + "license": "MIT" + }, + "node_modules/@algolia/ingestion": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/@algolia/ingestion/-/ingestion-1.19.0.tgz", + "integrity": "sha512-LO7w1MDV+ZLESwfPmXkp+KLeYeFrYEgtbCZG6buWjddhYraPQ9MuQWLhLLiaMlKxZ/sZvFTcZYuyI6Jx4WBhcg==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/ingestion/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/ingestion/node_modules/@algolia/requester-browser-xhr": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.19.0.tgz", + "integrity": "sha512-GfnhnQBT23mW/VMNs7m1qyEyZzhZz093aY2x8p0era96MMyNv8+FxGek5pjVX0b57tmSCZPf4EqNCpkGcGsmbw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/ingestion/node_modules/@algolia/requester-node-http": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.19.0.tgz", + "integrity": "sha512-p6t8ue0XZNjcRiqNkb5QAM0qQRAKsCiebZ6n9JjWA+p8fWf8BvnhO55y2fO28g3GW0Imj7PrAuyBuxq8aDVQwQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.24.0.tgz", + "integrity": "sha512-LLUNjkahj9KtKYrQhFKCzMx0BY3RnNP4FEtO+sBybCjJ73E8jNdaKJ/Dd8A/VA4imVHP5tADZ8pn5B8Ga/wTMA==", + "license": "MIT" + }, + "node_modules/@algolia/logger-console": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.24.0.tgz", + "integrity": "sha512-X4C8IoHgHfiUROfoRCV+lzSy+LHMgkoEEU1BbKcsfnV0i0S20zyy0NLww9dwVHUWNfPPxdMU+/wKmLGYf96yTg==", + "license": "MIT", + "dependencies": { + "@algolia/logger-common": "4.24.0" + } + }, + "node_modules/@algolia/monitoring": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/@algolia/monitoring/-/monitoring-1.19.0.tgz", + "integrity": "sha512-Mg4uoS0aIKeTpu6iv6O0Hj81s8UHagi5TLm9k2mLIib4vmMtX7WgIAHAcFIaqIZp5D6s5EVy1BaDOoZ7buuJHA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/monitoring/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/monitoring/node_modules/@algolia/requester-browser-xhr": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.19.0.tgz", + "integrity": "sha512-GfnhnQBT23mW/VMNs7m1qyEyZzhZz093aY2x8p0era96MMyNv8+FxGek5pjVX0b57tmSCZPf4EqNCpkGcGsmbw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/monitoring/node_modules/@algolia/requester-node-http": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.19.0.tgz", + "integrity": "sha512-p6t8ue0XZNjcRiqNkb5QAM0qQRAKsCiebZ6n9JjWA+p8fWf8BvnhO55y2fO28g3GW0Imj7PrAuyBuxq8aDVQwQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/recommend": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/recommend/-/recommend-4.24.0.tgz", + "integrity": "sha512-P9kcgerfVBpfYHDfVZDvvdJv0lEoCvzNlOy2nykyt5bK8TyieYyiD0lguIJdRZZYGre03WIAFf14pgE+V+IBlw==", + "license": "MIT", + "dependencies": { + "@algolia/cache-browser-local-storage": "4.24.0", + "@algolia/cache-common": "4.24.0", + "@algolia/cache-in-memory": "4.24.0", + "@algolia/client-common": "4.24.0", + "@algolia/client-search": "4.24.0", + "@algolia/logger-common": "4.24.0", + "@algolia/logger-console": "4.24.0", + "@algolia/requester-browser-xhr": "4.24.0", + "@algolia/requester-common": "4.24.0", + "@algolia/requester-node-http": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.24.0.tgz", + "integrity": "sha512-Z2NxZMb6+nVXSjF13YpjYTdvV3032YTBSGm2vnYvYPA6mMxzM3v5rsCiSspndn9rzIW4Qp1lPHBvuoKJV6jnAA==", + "license": "MIT", + "dependencies": { + "@algolia/requester-common": "4.24.0" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.24.0.tgz", + "integrity": "sha512-k3CXJ2OVnvgE3HMwcojpvY6d9kgKMPRxs/kVohrwF5WMr2fnqojnycZkxPoEg+bXm8fi5BBfFmOqgYztRtHsQA==", + "license": "MIT" + }, + "node_modules/@algolia/requester-fetch": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-fetch/-/requester-fetch-5.19.0.tgz", + "integrity": "sha512-oyTt8ZJ4T4fYvW5avAnuEc6Laedcme9fAFryMD9ndUTIUe/P0kn3BuGcCLFjN3FDmdrETHSFkgPPf1hGy3sLCw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/requester-fetch/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.24.0.tgz", + "integrity": "sha512-JF18yTjNOVYvU/L3UosRcvbPMGT9B+/GQWNWnenIImglzNVGpyzChkXLnrSf6uxwVNO6ESGu6oN8MqcGQcjQJw==", + "license": "MIT", + "dependencies": { + "@algolia/requester-common": "4.24.0" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.24.0.tgz", + "integrity": "sha512-86nI7w6NzWxd1Zp9q3413dRshDqAzSbsQjhcDhPIatEFiZrL1/TjnHL8S7jVKFePlIMzDsZWXAXwXzcok9c5oA==", + "license": "MIT", + "dependencies": { + "@algolia/cache-common": "4.24.0", + "@algolia/logger-common": "4.24.0", + "@algolia/requester-common": "4.24.0" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@antfu/install-pkg": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-0.4.1.tgz", + "integrity": "sha512-T7yB5QNG29afhWVkVq7XeIMBa5U/vs9mX69YqayXypPRmYzUmzwnYltplHmPtZ4HPCn+sQKeXW8I47wCbuBOjw==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^0.2.0", + "tinyexec": "^0.3.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@antfu/utils": { + "version": "0.7.10", + "resolved": "https://registry.npmjs.org/@antfu/utils/-/utils-0.7.10.tgz", + "integrity": "sha512-+562v9k4aI80m1+VuMHehNJWLOFjBnXn3tdOitzD0il5b7smkSBal4+a3oKiQTbrwMmN/TBUMDvbdoWDehgOww==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.2.tgz", + "integrity": "sha512-Z0WgzSEa+aUcdiJuCIqgujCshpMWgUpgOxXotrYPSA53hA3qopNaqcJpyr0hVb1FeWdnqFA35/fUtXgBK8srQg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", + "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.26.0", + "@babel/generator": "^7.26.0", + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helpers": "^7.26.0", + "@babel/parser": "^7.26.0", + "@babel/template": "^7.25.9", + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.26.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", + "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.26.2", + "@babel/types": "^7.26.0", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.25.9.tgz", + "integrity": "sha512-gv7320KBUFJz1RnylIg5WWYPRXKZ884AGkYpgpWW02TH66Dl+HaC1t1CKd0z3R4b6hdYEcmrNZHUmfCP+1u3/g==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.25.9.tgz", + "integrity": "sha512-C47lC7LIDCnz0h4vai/tpNOI95tCd5ZT3iBt/DBH5lXKHZsyNQv18yf1wIIg2ntiQNgmAvA+DgZ82iW8Qdym8g==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", + "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.25.9.tgz", + "integrity": "sha512-UTZQMvt0d/rSz6KI+qdu7GQze5TIajwTS++GUozlw8VBJDEOAqSXwm1WvmYEZwqdqSGQshRocPDqrt4HBZB3fQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-member-expression-to-functions": "^7.25.9", + "@babel/helper-optimise-call-expression": "^7.25.9", + "@babel/helper-replace-supers": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", + "@babel/traverse": "^7.25.9", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-class-features-plugin/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.25.9.tgz", + "integrity": "sha512-ORPNZ3h6ZRkOyAa/SaHU+XsLZr0UQzRwuDQ0cczIA17nAzZ+85G5cVkOJIj7QavLZGSe8QXUmNFxSZzjcZF9bw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "regexpu-core": "^6.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-define-polyfill-provider": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.6.3.tgz", + "integrity": "sha512-HK7Bi+Hj6H+VTHA3ZvBis7V/6hu9QuTrnMXNybfUf2iiuU/N97I8VjB+KbhFF8Rld/Lx5MzoCwPCpPjfK+n8Cg==", + "license": "MIT", + "dependencies": { + "@babel/helper-compilation-targets": "^7.22.6", + "@babel/helper-plugin-utils": "^7.22.5", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.25.9.tgz", + "integrity": "sha512-wbfdZ9w5vk0C0oyHqAJbc62+vet5prjj01jjJ8sKn3j9h3MQQlflEdXYvuqRWjHnM12coDEqiC1IRCi0U/EKwQ==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", + "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.25.9.tgz", + "integrity": "sha512-FIpuNaz5ow8VyrYcnXQTDRGvV6tTjkNtCK/RYNDXGSLlUD6cBuQTSw43CShGxjvfBTfcUA/r6UhUCbtYqkhcuQ==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", + "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-remap-async-to-generator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.25.9.tgz", + "integrity": "sha512-IZtukuUeBbhgOcaW2s06OXTzVNJR0ybm4W5xC1opWFFJMZbwRj5LCk+ByYH7WdZPZTt8KnFwA8pvjN2yqcPlgw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-wrap-function": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.25.9.tgz", + "integrity": "sha512-IiDqTOTBQy0sWyeXyGSC5TBJpGFXBkRynjBeXsvbhQFKj2viwJC76Epz35YLU1fpe/Am6Vppb7W7zM4fPQzLsQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-member-expression-to-functions": "^7.25.9", + "@babel/helper-optimise-call-expression": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.25.9.tgz", + "integrity": "sha512-c6WHXuiaRsJTyHYLJV75t9IqsmTbItYfdj99PnzYGQZkYKvan5/2jKJ7gu31J3/BJ/A18grImSPModuyG/Eo0Q==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.25.9.tgz", + "integrity": "sha512-K4Du3BFa3gvyhzgPcntrkDgZzQaq6uozzcpGbOO1OEJaI+EJdqWIMTLgFgQf6lrfiDFo5FU+BxKepI9RmZqahA==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", + "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-wrap-function": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.25.9.tgz", + "integrity": "sha512-ETzz9UTjQSTmw39GboatdymDq4XIQbR8ySgVrylRhPOFpsd+JrKHIuF0de7GCWmem+T4uC5z7EZguod7Wj4A4g==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.25.9", + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", + "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz", + "integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-firefox-class-in-computed-class-key": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-firefox-class-in-computed-class-key/-/plugin-bugfix-firefox-class-in-computed-class-key-7.25.9.tgz", + "integrity": "sha512-ZkRyVkThtxQ/J6nv3JFYv1RYY+JT5BvU0y3k5bWrmuG4woXypRa4PXmm9RhOwodRkYFWqC0C0cqcJ4OqR7kW+g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-class-field-initializer-scope": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-class-field-initializer-scope/-/plugin-bugfix-safari-class-field-initializer-scope-7.25.9.tgz", + "integrity": "sha512-MrGRLZxLD/Zjj0gdU15dfs+HH/OXvnw/U4jJD8vpcP2CJQapPEv1IWwjc/qMg7ItBlPwSv1hRBbb7LeuANdcnw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.25.9.tgz", + "integrity": "sha512-2qUwwfAFpJLZqxd02YW9btUCZHl+RFvdDkNfZwaIJrvB8Tesjsk8pEQkTvGwZXLqXUx/2oyY3ySRhm6HOXuCug==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.25.9.tgz", + "integrity": "sha512-6xWgLZTJXwilVjlnV7ospI3xi+sl8lN8rXXbBD6vYn3UYDlGsag8wrZkKcSI8G6KgqKP7vNFaDgeDnfAABq61g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", + "@babel/plugin-transform-optional-chaining": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.13.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly/-/plugin-bugfix-v8-static-class-fields-redefine-readonly-7.25.9.tgz", + "integrity": "sha512-aLnMXYPnzwwqhYSCyXfKkIkYgJ8zv9RK+roo9DkTXz38ynIhd9XCbN08s3MGvqL2MYGVUGdRQLL/JqBIeJhJBg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-proposal-private-property-in-object": { + "version": "7.21.0-placeholder-for-preset-env.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.0-placeholder-for-preset-env.2.tgz", + "integrity": "sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-assertions": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.26.0.tgz", + "integrity": "sha512-QCWT5Hh830hK5EQa7XzuqIkQU9tT/whqbDz7kuaZMHFl1inRRg7JnuAEOQ0Ur0QUl0NufCk1msK2BeY79Aj/eg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", + "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", + "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", + "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-unicode-sets-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-unicode-sets-regex/-/plugin-syntax-unicode-sets-regex-7.18.6.tgz", + "integrity": "sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.25.9.tgz", + "integrity": "sha512-6jmooXYIwn9ca5/RylZADJ+EnSxVUS5sjeJ9UPk6RWRzXCmOJCy6dqItPJFpw2cuCangPK4OYr5uhGKcmrm5Qg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-generator-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.25.9.tgz", + "integrity": "sha512-RXV6QAzTBbhDMO9fWwOmwwTuYaiPbggWQ9INdZqAYeSHyG7FzQ+nOZaUUjNwKv9pV3aE4WFqFm1Hnbci5tBCAw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-remap-async-to-generator": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-to-generator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.25.9.tgz", + "integrity": "sha512-NT7Ejn7Z/LjUH0Gv5KsBCxh7BH3fbLTV0ptHvpeMvrt3cPThHfJfst9Wrb7S8EvJ7vRTFI7z+VAvFVEQn/m5zQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-remap-async-to-generator": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoped-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.25.9.tgz", + "integrity": "sha512-toHc9fzab0ZfenFpsyYinOX0J/5dgJVA2fm64xPewu7CoYHWEivIWKxkK2rMi4r3yQqLnVmheMXRdG+k239CgA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoping": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.25.9.tgz", + "integrity": "sha512-1F05O7AYjymAtqbsFETboN1NvBdcnzMerO+zlMyJBEz6WkMdejvGWw9p05iTSjC85RLlBseHHQpYaM4gzJkBGg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-class-properties": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-properties/-/plugin-transform-class-properties-7.25.9.tgz", + "integrity": "sha512-bbMAII8GRSkcd0h0b4X+36GksxuheLFjP65ul9w6C3KgAamI3JqErNgSrosX6ZPj+Mpim5VvEbawXxJCyEUV3Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-class-static-block": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-static-block/-/plugin-transform-class-static-block-7.26.0.tgz", + "integrity": "sha512-6J2APTs7BDDm+UMqP1useWqhcRAXo0WIoVj26N7kPFB6S73Lgvyka4KTZYIxtgYXiN5HTyRObA72N2iu628iTQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.12.0" + } + }, + "node_modules/@babel/plugin-transform-classes": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.25.9.tgz", + "integrity": "sha512-mD8APIXmseE7oZvZgGABDyM34GUmK45Um2TXiBUt7PnuAxrgoSVf123qUzPxEr/+/BHrRn5NMZCdE2m/1F8DGg==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-replace-supers": "^7.25.9", + "@babel/traverse": "^7.25.9", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-computed-properties": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.25.9.tgz", + "integrity": "sha512-HnBegGqXZR12xbcTHlJ9HGxw1OniltT26J5YpfruGqtUHlz/xKf/G2ak9e+t0rVqrjXa9WOhvYPz1ERfMj23AA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/template": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-destructuring": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.25.9.tgz", + "integrity": "sha512-WkCGb/3ZxXepmMiX101nnGiU+1CAdut8oHyEOHxkKuS1qKpU2SMXE2uSvfz8PBuLd49V6LEsbtyPhWC7fnkgvQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-dotall-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.25.9.tgz", + "integrity": "sha512-t7ZQ7g5trIgSRYhI9pIJtRl64KHotutUJsh4Eze5l7olJv+mRSg4/MmbZ0tv1eeqRbdvo/+trvJD/Oc5DmW2cA==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-keys": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.25.9.tgz", + "integrity": "sha512-LZxhJ6dvBb/f3x8xwWIuyiAHy56nrRG3PeYTpBkkzkYRRQ6tJLu68lEF5VIqMUZiAV7a8+Tb78nEoMCMcqjXBw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-named-capturing-groups-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-named-capturing-groups-regex/-/plugin-transform-duplicate-named-capturing-groups-regex-7.25.9.tgz", + "integrity": "sha512-0UfuJS0EsXbRvKnwcLjFtJy/Sxc5J5jhLHnFhy7u4zih97Hz6tJkLU+O+FMMrNZrosUPxDi6sYxJ/EA8jDiAog==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-dynamic-import": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dynamic-import/-/plugin-transform-dynamic-import-7.25.9.tgz", + "integrity": "sha512-GCggjexbmSLaFhqsojeugBpeaRIgWNTcgKVq/0qIteFEqY2A+b9QidYadrWlnbWQUrW5fn+mCvf3tr7OeBFTyg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-exponentiation-operator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.25.9.tgz", + "integrity": "sha512-KRhdhlVk2nObA5AYa7QMgTMTVJdfHprfpAk4DjZVtllqRg9qarilstTKEhpVjyt+Npi8ThRyiV8176Am3CodPA==", + "license": "MIT", + "dependencies": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-export-namespace-from": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-export-namespace-from/-/plugin-transform-export-namespace-from-7.25.9.tgz", + "integrity": "sha512-2NsEz+CxzJIVOPx2o9UsW1rXLqtChtLoVnwYHHiB04wS5sgn7mrV45fWMBX0Kk+ub9uXytVYfNP2HjbVbCB3Ww==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-for-of": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.25.9.tgz", + "integrity": "sha512-LqHxduHoaGELJl2uhImHwRQudhCM50pT46rIBNvtT/Oql3nqiS3wOwP+5ten7NpYSXrrVLgtZU3DZmPtWZo16A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-function-name": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.25.9.tgz", + "integrity": "sha512-8lP+Yxjv14Vc5MuWBpJsoUCd3hD6V9DgBon2FVYL4jJgbnVQ9fTgYmonchzZJOVNgzEgbxp4OwAf6xz6M/14XA==", + "license": "MIT", + "dependencies": { + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-json-strings": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-json-strings/-/plugin-transform-json-strings-7.25.9.tgz", + "integrity": "sha512-xoTMk0WXceiiIvsaquQQUaLLXSW1KJ159KP87VilruQm0LNNGxWzahxSS6T6i4Zg3ezp4vA4zuwiNUR53qmQAw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.25.9.tgz", + "integrity": "sha512-9N7+2lFziW8W9pBl2TzaNht3+pgMIRP74zizeCSrtnSKVdUl8mAjjOP2OOVQAfZ881P2cNjDj1uAMEdeD50nuQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-logical-assignment-operators": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-logical-assignment-operators/-/plugin-transform-logical-assignment-operators-7.25.9.tgz", + "integrity": "sha512-wI4wRAzGko551Y8eVf6iOY9EouIDTtPb0ByZx+ktDGHwv6bHFimrgJM/2T021txPZ2s4c7bqvHbd+vXG6K948Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-member-expression-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.25.9.tgz", + "integrity": "sha512-PYazBVfofCQkkMzh2P6IdIUaCEWni3iYEerAsRWuVd8+jlM1S9S9cz1dF9hIzyoZ8IA3+OwVYIp9v9e+GbgZhA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-amd": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.25.9.tgz", + "integrity": "sha512-g5T11tnI36jVClQlMlt4qKDLlWnG5pP9CSM4GhdRciTNMRgkfpo5cR6b4rGIOYPgRRuFAvwjPQ/Yk+ql4dyhbw==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.25.9.tgz", + "integrity": "sha512-dwh2Ol1jWwL2MgkCzUSOvfmKElqQcuswAZypBSUsScMXvgdT8Ekq5YA6TtqpTVWH+4903NmboMuH1o9i8Rxlyg==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-simple-access": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-systemjs": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.25.9.tgz", + "integrity": "sha512-hyss7iIlH/zLHaehT+xwiymtPOpsiwIIRlCAOwBB04ta5Tt+lNItADdlXw3jAWZ96VJ2jlhl/c+PNIQPKNfvcA==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-umd": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.25.9.tgz", + "integrity": "sha512-bS9MVObUgE7ww36HEfwe6g9WakQ0KF07mQF74uuXdkoziUPfKyu/nIm663kz//e5O1nPInPFx36z7WJmJ4yNEw==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.25.9.tgz", + "integrity": "sha512-oqB6WHdKTGl3q/ItQhpLSnWWOpjUJLsOCLVyeFgeTktkBSCiurvPOsyt93gibI9CmuKvTUEtWmG5VhZD+5T/KA==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-new-target": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.25.9.tgz", + "integrity": "sha512-U/3p8X1yCSoKyUj2eOBIx3FOn6pElFOKvAAGf8HTtItuPyB+ZeOqfn+mvTtg9ZlOAjsPdK3ayQEjqHjU/yLeVQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-nullish-coalescing-operator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.25.9.tgz", + "integrity": "sha512-ENfftpLZw5EItALAD4WsY/KUWvhUlZndm5GC7G3evUsVeSJB6p0pBeLQUnRnBCBx7zV0RKQjR9kCuwrsIrjWog==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-numeric-separator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-numeric-separator/-/plugin-transform-numeric-separator-7.25.9.tgz", + "integrity": "sha512-TlprrJ1GBZ3r6s96Yq8gEQv82s8/5HnCVHtEJScUj90thHQbwe+E5MLhi2bbNHBEJuzrvltXSru+BUxHDoog7Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-rest-spread": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-rest-spread/-/plugin-transform-object-rest-spread-7.25.9.tgz", + "integrity": "sha512-fSaXafEE9CVHPweLYw4J0emp1t8zYTXyzN3UuG+lylqkvYd7RMrsOQ8TYx5RF231be0vqtFC6jnx3UmpJmKBYg==", + "license": "MIT", + "dependencies": { + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/plugin-transform-parameters": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-super": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.25.9.tgz", + "integrity": "sha512-Kj/Gh+Rw2RNLbCK1VAWj2U48yxxqL2x0k10nPtSdRa0O2xnHXalD0s+o1A6a0W43gJ00ANo38jxkQreckOzv5A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-replace-supers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-optional-catch-binding": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-catch-binding/-/plugin-transform-optional-catch-binding-7.25.9.tgz", + "integrity": "sha512-qM/6m6hQZzDcZF3onzIhZeDHDO43bkNNlOX0i8n3lR6zLbu0GN2d8qfM/IERJZYauhAHSLHy39NF0Ctdvcid7g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-optional-chaining": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.25.9.tgz", + "integrity": "sha512-6AvV0FsLULbpnXeBjrY4dmWF8F7gf8QnvTEoO/wX/5xm/xE1Xo8oPuD3MPS+KS9f9XBEAWN7X1aWr4z9HdOr7A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-parameters": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.25.9.tgz", + "integrity": "sha512-wzz6MKwpnshBAiRmn4jR8LYz/g8Ksg0o80XmwZDlordjwEk9SxBzTWC7F5ef1jhbrbOW2DJ5J6ayRukrJmnr0g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-private-methods": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-methods/-/plugin-transform-private-methods-7.25.9.tgz", + "integrity": "sha512-D/JUozNpQLAPUVusvqMxyvjzllRaF8/nSrP1s2YGQT/W4LHK4xxsMcHjhOGTS01mp9Hda8nswb+FblLdJornQw==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-private-property-in-object": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-property-in-object/-/plugin-transform-private-property-in-object-7.25.9.tgz", + "integrity": "sha512-Evf3kcMqzXA3xfYJmZ9Pg1OvKdtqsDMSWBDzZOPLvHiTt36E75jLDQo5w1gtRU95Q4E5PDttrTf25Fw8d/uWLw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-property-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.25.9.tgz", + "integrity": "sha512-IvIUeV5KrS/VPavfSM/Iu+RE6llrHrYIKY1yfCzyO/lMXHQ+p7uGhonmGVisv6tSBSVgWzMBohTcvkC9vQcQFA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-constant-elements": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-constant-elements/-/plugin-transform-react-constant-elements-7.25.9.tgz", + "integrity": "sha512-Ncw2JFsJVuvfRsa2lSHiC55kETQVLSnsYGQ1JDDwkUeWGTL/8Tom8aLTnlqgoeuopWrbbGndrc9AlLYrIosrow==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-display-name": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.25.9.tgz", + "integrity": "sha512-KJfMlYIUxQB1CJfO3e0+h0ZHWOTLCPP115Awhaz8U0Zpq36Gl/cXlpoyMRnUWlhNUBAzldnCiAZNvCDj7CrKxQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.25.9.tgz", + "integrity": "sha512-s5XwpQYCqGerXl+Pu6VDL3x0j2d82eiV77UJ8a2mDHAW7j9SWRqQ2y1fNo1Z74CdcYipl5Z41zvjj4Nfzq36rw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/plugin-syntax-jsx": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-development": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.25.9.tgz", + "integrity": "sha512-9mj6rm7XVYs4mdLIpbZnHOYdpW42uoiBCTVowg7sP1thUOiANgMb4UtpRivR0pp5iL+ocvUv7X4mZgFRpJEzGw==", + "license": "MIT", + "dependencies": { + "@babel/plugin-transform-react-jsx": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-pure-annotations": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.25.9.tgz", + "integrity": "sha512-KQ/Takk3T8Qzj5TppkS1be588lkbTp5uj7w6a0LeQaTMSckU/wK0oJ/pih+T690tkgI5jfmg2TqDJvd41Sj1Cg==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.25.9.tgz", + "integrity": "sha512-vwDcDNsgMPDGP0nMqzahDWE5/MLcX8sv96+wfX7as7LoF/kr97Bo/7fI00lXY4wUXYfVmwIIyG80fGZ1uvt2qg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "regenerator-transform": "^0.15.2" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regexp-modifiers": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regexp-modifiers/-/plugin-transform-regexp-modifiers-7.26.0.tgz", + "integrity": "sha512-vN6saax7lrA2yA/Pak3sCxuD6F5InBjn9IcrIKQPjpsLvuHYLVroTxjdlVRHjjBWxKOqIwpTXDkOssYT4BFdRw==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-reserved-words": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.25.9.tgz", + "integrity": "sha512-7DL7DKYjn5Su++4RXu8puKZm2XBPHyjWLUidaPEkCUBbE7IPcsrkRHggAOOKydH1dASWdcUBxrkOGNxUv5P3Jg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-runtime": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.25.9.tgz", + "integrity": "sha512-nZp7GlEl+yULJrClz0SwHPqir3lc0zsPrDHQUcxGspSL7AKrexNSEfTbfqnDNJUO13bgKyfuOLMF8Xqtu8j3YQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "babel-plugin-polyfill-corejs2": "^0.4.10", + "babel-plugin-polyfill-corejs3": "^0.10.6", + "babel-plugin-polyfill-regenerator": "^0.6.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-runtime/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.25.9.tgz", + "integrity": "sha512-MUv6t0FhO5qHnS/W8XCbHmiRWOphNufpE1IVxhK5kuN3Td9FT1x4rx4K42s3RYdMXCXpfWkGSbCSd0Z64xA7Ng==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-spread": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.25.9.tgz", + "integrity": "sha512-oNknIB0TbURU5pqJFVbOOFspVlrpVwo2H1+HUIsVDvp5VauGGDP1ZEvO8Nn5xyMEs3dakajOxlmkNW7kNgSm6A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-sticky-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.25.9.tgz", + "integrity": "sha512-WqBUSgeVwucYDP9U/xNRQam7xV8W5Zf+6Eo7T2SRVUFlhRiMNFdFz58u0KZmCVVqs2i7SHgpRnAhzRNmKfi2uA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.25.9.tgz", + "integrity": "sha512-o97AE4syN71M/lxrCtQByzphAdlYluKPDBzDVzMmfCobUjjhAryZV0AIpRPrxN0eAkxXO6ZLEScmt+PNhj2OTw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typeof-symbol": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.25.9.tgz", + "integrity": "sha512-v61XqUMiueJROUv66BVIOi0Fv/CUuZuZMl5NkRoCVxLAnMexZ0A3kMe7vvZ0nulxMuMp0Mk6S5hNh48yki08ZA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typescript": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.25.9.tgz", + "integrity": "sha512-7PbZQZP50tzv2KGGnhh82GSyMB01yKY9scIjf1a+GfZCtInOWqUH5+1EBU4t9fyR5Oykkkc9vFTs4OHrhHXljQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", + "@babel/plugin-syntax-typescript": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-escapes": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.25.9.tgz", + "integrity": "sha512-s5EDrE6bW97LtxOcGj1Khcx5AaXwiMmi4toFWRDP9/y0Woo6pXC+iyPu/KuhKtfSrNFd7jJB+/fkOtZy6aIC6Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-property-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-property-regex/-/plugin-transform-unicode-property-regex-7.25.9.tgz", + "integrity": "sha512-Jt2d8Ga+QwRluxRQ307Vlxa6dMrYEMZCgGxoPR8V52rxPyldHu3hdlHspxaqYmE7oID5+kB+UKUB/eWS+DkkWg==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.25.9.tgz", + "integrity": "sha512-yoxstj7Rg9dlNn9UQxzk4fcNivwv4nUYz7fYXBaKxvw/lnmPuOm/ikoELygbYq68Bls3D/D+NBPHiLwZdZZ4HA==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-sets-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-sets-regex/-/plugin-transform-unicode-sets-regex-7.25.9.tgz", + "integrity": "sha512-8BYqO3GeVNHtx69fdPshN3fnzUNLrWdHhk/icSwigksJGczKSizZ+Z6SBCxTs723Fr5VSNorTIK7a+R2tISvwQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/preset-env": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.26.0.tgz", + "integrity": "sha512-H84Fxq0CQJNdPFT2DrfnylZ3cf5K43rGfWK4LJGPpjKHiZlk0/RzwEus3PDDZZg+/Er7lCA03MVacueUuXdzfw==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.26.0", + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-bugfix-firefox-class-in-computed-class-key": "^7.25.9", + "@babel/plugin-bugfix-safari-class-field-initializer-scope": "^7.25.9", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.25.9", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.25.9", + "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": "^7.25.9", + "@babel/plugin-proposal-private-property-in-object": "7.21.0-placeholder-for-preset-env.2", + "@babel/plugin-syntax-import-assertions": "^7.26.0", + "@babel/plugin-syntax-import-attributes": "^7.26.0", + "@babel/plugin-syntax-unicode-sets-regex": "^7.18.6", + "@babel/plugin-transform-arrow-functions": "^7.25.9", + "@babel/plugin-transform-async-generator-functions": "^7.25.9", + "@babel/plugin-transform-async-to-generator": "^7.25.9", + "@babel/plugin-transform-block-scoped-functions": "^7.25.9", + "@babel/plugin-transform-block-scoping": "^7.25.9", + "@babel/plugin-transform-class-properties": "^7.25.9", + "@babel/plugin-transform-class-static-block": "^7.26.0", + "@babel/plugin-transform-classes": "^7.25.9", + "@babel/plugin-transform-computed-properties": "^7.25.9", + "@babel/plugin-transform-destructuring": "^7.25.9", + "@babel/plugin-transform-dotall-regex": "^7.25.9", + "@babel/plugin-transform-duplicate-keys": "^7.25.9", + "@babel/plugin-transform-duplicate-named-capturing-groups-regex": "^7.25.9", + "@babel/plugin-transform-dynamic-import": "^7.25.9", + "@babel/plugin-transform-exponentiation-operator": "^7.25.9", + "@babel/plugin-transform-export-namespace-from": "^7.25.9", + "@babel/plugin-transform-for-of": "^7.25.9", + "@babel/plugin-transform-function-name": "^7.25.9", + "@babel/plugin-transform-json-strings": "^7.25.9", + "@babel/plugin-transform-literals": "^7.25.9", + "@babel/plugin-transform-logical-assignment-operators": "^7.25.9", + "@babel/plugin-transform-member-expression-literals": "^7.25.9", + "@babel/plugin-transform-modules-amd": "^7.25.9", + "@babel/plugin-transform-modules-commonjs": "^7.25.9", + "@babel/plugin-transform-modules-systemjs": "^7.25.9", + "@babel/plugin-transform-modules-umd": "^7.25.9", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.25.9", + "@babel/plugin-transform-new-target": "^7.25.9", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.25.9", + "@babel/plugin-transform-numeric-separator": "^7.25.9", + "@babel/plugin-transform-object-rest-spread": "^7.25.9", + "@babel/plugin-transform-object-super": "^7.25.9", + "@babel/plugin-transform-optional-catch-binding": "^7.25.9", + "@babel/plugin-transform-optional-chaining": "^7.25.9", + "@babel/plugin-transform-parameters": "^7.25.9", + "@babel/plugin-transform-private-methods": "^7.25.9", + "@babel/plugin-transform-private-property-in-object": "^7.25.9", + "@babel/plugin-transform-property-literals": "^7.25.9", + "@babel/plugin-transform-regenerator": "^7.25.9", + "@babel/plugin-transform-regexp-modifiers": "^7.26.0", + "@babel/plugin-transform-reserved-words": "^7.25.9", + "@babel/plugin-transform-shorthand-properties": "^7.25.9", + "@babel/plugin-transform-spread": "^7.25.9", + "@babel/plugin-transform-sticky-regex": "^7.25.9", + "@babel/plugin-transform-template-literals": "^7.25.9", + "@babel/plugin-transform-typeof-symbol": "^7.25.9", + "@babel/plugin-transform-unicode-escapes": "^7.25.9", + "@babel/plugin-transform-unicode-property-regex": "^7.25.9", + "@babel/plugin-transform-unicode-regex": "^7.25.9", + "@babel/plugin-transform-unicode-sets-regex": "^7.25.9", + "@babel/preset-modules": "0.1.6-no-external-plugins", + "babel-plugin-polyfill-corejs2": "^0.4.10", + "babel-plugin-polyfill-corejs3": "^0.10.6", + "babel-plugin-polyfill-regenerator": "^0.6.1", + "core-js-compat": "^3.38.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/preset-modules": { + "version": "0.1.6-no-external-plugins", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.6-no-external-plugins.tgz", + "integrity": "sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/@babel/preset-react": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.25.9.tgz", + "integrity": "sha512-D3to0uSPiWE7rBrdIICCd0tJSIGpLaaGptna2+w7Pft5xMqLpA1sz99DK5TZ1TjGbdQ/VI1eCSZ06dv3lT4JOw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-transform-react-display-name": "^7.25.9", + "@babel/plugin-transform-react-jsx": "^7.25.9", + "@babel/plugin-transform-react-jsx-development": "^7.25.9", + "@babel/plugin-transform-react-pure-annotations": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-typescript": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.26.0.tgz", + "integrity": "sha512-NMk1IGZ5I/oHhoXEElcm+xUnL/szL6xflkFZmoEU9xj1qSJXpiS7rsspYo92B4DRCDvZn2erT5LdsCeXAKNCkg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-syntax-jsx": "^7.25.9", + "@babel/plugin-transform-modules-commonjs": "^7.25.9", + "@babel/plugin-transform-typescript": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/runtime-corejs3": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.26.0.tgz", + "integrity": "sha512-YXHu5lN8kJCb1LOb9PgV6pvak43X2h4HvRApcN5SdWeaItQOzfn1hgP6jasD6KWQyJDBxrVmA9o9OivlnNJK/w==", + "license": "MIT", + "dependencies": { + "core-js-pure": "^3.30.2", + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", + "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/generator": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/template": "^7.25.9", + "@babel/types": "^7.25.9", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", + "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@braintree/sanitize-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.0.tgz", + "integrity": "sha512-o+UlMLt49RvtCASlOMW0AkHnabN9wR9rwCCherxO0yG4Npy34GkvrAqdXQvrhNs+jh+gkK8gB8Lf05qL/O7KWg==", + "license": "MIT" + }, + "node_modules/@chevrotain/cst-dts-gen": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/cst-dts-gen/-/cst-dts-gen-11.0.3.tgz", + "integrity": "sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ==", + "license": "Apache-2.0", + "dependencies": { + "@chevrotain/gast": "11.0.3", + "@chevrotain/types": "11.0.3", + "lodash-es": "4.17.21" + } + }, + "node_modules/@chevrotain/gast": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/gast/-/gast-11.0.3.tgz", + "integrity": "sha512-+qNfcoNk70PyS/uxmj3li5NiECO+2YKZZQMbmjTqRI3Qchu8Hig/Q9vgkHpI3alNjr7M+a2St5pw5w5F6NL5/Q==", + "license": "Apache-2.0", + "dependencies": { + "@chevrotain/types": "11.0.3", + "lodash-es": "4.17.21" + } + }, + "node_modules/@chevrotain/regexp-to-ast": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/regexp-to-ast/-/regexp-to-ast-11.0.3.tgz", + "integrity": "sha512-1fMHaBZxLFvWI067AVbGJav1eRY7N8DDvYCTwGBiE/ytKBgP8azTdgyrKyWZ9Mfh09eHWb5PgTSO8wi7U824RA==", + "license": "Apache-2.0" + }, + "node_modules/@chevrotain/types": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.0.3.tgz", + "integrity": "sha512-gsiM3G8b58kZC2HaWR50gu6Y1440cHiJ+i3JUvcp/35JchYejb2+5MVeJK0iKThYpAa/P2PYFV4hoi44HD+aHQ==", + "license": "Apache-2.0" + }, + "node_modules/@chevrotain/utils": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/utils/-/utils-11.0.3.tgz", + "integrity": "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==", + "license": "Apache-2.0" + }, + "node_modules/@cmfcmf/docusaurus-search-local": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@cmfcmf/docusaurus-search-local/-/docusaurus-search-local-1.2.0.tgz", + "integrity": "sha512-Tc0GhRBsfZAiB+f6BoPB8YCQap6JzzcDyJ0dLSCSzWQ6wdWvDlTBrHc1YqR8q8AZ+STRszL5eZpZFi5dbTCdYg==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-js": "^1.8.2", + "@algolia/autocomplete-theme-classic": "^1.8.2", + "@algolia/client-search": "^4.12.0", + "algoliasearch": "^4.12.0", + "cheerio": "^1.0.0-rc.9", + "clsx": "^1.1.1", + "lunr-languages": "^1.4.0", + "mark.js": "^8.11.1", + "tslib": "^2.6.3" + }, + "peerDependencies": { + "@docusaurus/core": "^2.0.0", + "nodejieba": "^2.5.0" + }, + "peerDependenciesMeta": { + "nodejieba": { + "optional": true + } + } + }, + "node_modules/@cmfcmf/docusaurus-search-local/node_modules/clsx": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.2.1.tgz", + "integrity": "sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@csstools/cascade-layer-name-parser": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@csstools/cascade-layer-name-parser/-/cascade-layer-name-parser-2.0.4.tgz", + "integrity": "sha512-7DFHlPuIxviKYZrOiwVU/PiHLm3lLUR23OMuEEtfEOQTOp9hzQ2JjdY6X5H18RVuUPJqSCI+qNnD5iOLMVE0bA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.0.1.tgz", + "integrity": "sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.0.tgz", + "integrity": "sha512-X69PmFOrjTZfN5ijxtI8hZ9kRADFSLrmmQ6hgDJ272Il049WGKpDY64KhrFm/7rbWve0z81QepawzjkKlqkNGw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.0.6.tgz", + "integrity": "sha512-S/IjXqTHdpI4EtzGoNCHfqraXF37x12ZZHA1Lk7zoT5pm2lMjFuqhX/89L7dqX4CcMacKK+6ZCs5TmEGb/+wKw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.0.1", + "@csstools/css-calc": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.4.tgz", + "integrity": "sha512-Up7rBoV77rv29d3uKHUIVubz1BTcgyUK72IvCQAbfbMv584xHcGKCKbWh7i8hPrRJ7qU4Y8IO3IY9m+iTB7P3A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.3.tgz", + "integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/media-query-list-parser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/media-query-list-parser/-/media-query-list-parser-4.0.2.tgz", + "integrity": "sha512-EUos465uvVvMJehckATTlNqGj4UJWkTmdWuDMjqvSUkjGpmOyFZBVwb4knxCm/k2GMTXY+c/5RkdndzFYWeX5A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + } + }, + "node_modules/@csstools/postcss-cascade-layers": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@csstools/postcss-cascade-layers/-/postcss-cascade-layers-5.0.1.tgz", + "integrity": "sha512-XOfhI7GShVcKiKwmPAnWSqd2tBR0uxt+runAxttbSp/LY2U16yAVPmAf7e9q4JJ0d+xMNmpwNDLBXnmRCl3HMQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/selector-specificity": "^5.0.0", + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-cascade-layers/node_modules/@csstools/selector-specificity": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-5.0.0.tgz", + "integrity": "sha512-PCqQV3c4CoVm3kdPhyeZ07VmBRdH2EpMFA/pd9OASpOEC3aXNGoqPDAZ80D0cLpMBxnmk0+yNhGsEx31hq7Gtw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.0.0" + } + }, + "node_modules/@csstools/postcss-cascade-layers/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@csstools/postcss-color-function": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-color-function/-/postcss-color-function-4.0.6.tgz", + "integrity": "sha512-EcvXfC60cTIumzpsxWuvVjb7rsJEHPvqn3jeMEBUaE3JSc4FRuP7mEQ+1eicxWmIrs3FtzMH9gR3sgA5TH+ebQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-color-mix-function": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-color-mix-function/-/postcss-color-mix-function-3.0.6.tgz", + "integrity": "sha512-jVKdJn4+JkASYGhyPO+Wa5WXSx1+oUgaXb3JsjJn/BlrtFh5zjocCY7pwWi0nuP24V1fY7glQsxEYcYNy0dMFg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-content-alt-text": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@csstools/postcss-content-alt-text/-/postcss-content-alt-text-2.0.4.tgz", + "integrity": "sha512-YItlZUOuZJCBlRaCf8Aucc1lgN41qYGALMly0qQllrxYJhiyzlI6RxOTMUvtWk+KhS8GphMDsDhKQ7KTPfEMSw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-exponential-functions": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@csstools/postcss-exponential-functions/-/postcss-exponential-functions-2.0.5.tgz", + "integrity": "sha512-mi8R6dVfA2nDoKM3wcEi64I8vOYEgQVtVKCfmLHXupeLpACfGAided5ddMt5f+CnEodNu4DifuVwb0I6fQDGGQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-calc": "^2.1.0", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-font-format-keywords": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-font-format-keywords/-/postcss-font-format-keywords-4.0.0.tgz", + "integrity": "sha512-usBzw9aCRDvchpok6C+4TXC57btc4bJtmKQWOHQxOVKen1ZfVqBUuCZ/wuqdX5GHsD0NRSr9XTP+5ID1ZZQBXw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-gamut-mapping": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-gamut-mapping/-/postcss-gamut-mapping-2.0.6.tgz", + "integrity": "sha512-0ke7fmXfc8H+kysZz246yjirAH6JFhyX9GTlyRnM0exHO80XcA9zeJpy5pOp5zo/AZiC/q5Pf+Hw7Pd6/uAoYA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-gradients-interpolation-method": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-gradients-interpolation-method/-/postcss-gradients-interpolation-method-5.0.6.tgz", + "integrity": "sha512-Itrbx6SLUzsZ6Mz3VuOlxhbfuyLTogG5DwEF1V8dAi24iMuvQPIHd7Ti+pNDp7j6WixndJGZaoNR0f9VSzwuTg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-hwb-function": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-hwb-function/-/postcss-hwb-function-4.0.6.tgz", + "integrity": "sha512-927Pqy3a1uBP7U8sTfaNdZVB0mNXzIrJO/GZ8us9219q9n06gOqCdfZ0E6d1P66Fm0fYHvxfDbfcUuwAn5UwhQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-ic-unit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-ic-unit/-/postcss-ic-unit-4.0.0.tgz", + "integrity": "sha512-9QT5TDGgx7wD3EEMN3BSUG6ckb6Eh5gSPT5kZoVtUuAonfPmLDJyPhqR4ntPpMYhUKAMVKAg3I/AgzqHMSeLhA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-initial": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-initial/-/postcss-initial-2.0.0.tgz", + "integrity": "sha512-dv2lNUKR+JV+OOhZm9paWzYBXOCi+rJPqJ2cJuhh9xd8USVrd0cBEPczla81HNOyThMQWeCcdln3gZkQV2kYxA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-is-pseudo-class": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@csstools/postcss-is-pseudo-class/-/postcss-is-pseudo-class-5.0.1.tgz", + "integrity": "sha512-JLp3POui4S1auhDR0n8wHd/zTOWmMsmK3nQd3hhL6FhWPaox5W7j1se6zXOG/aP07wV2ww0lxbKYGwbBszOtfQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/selector-specificity": "^5.0.0", + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-is-pseudo-class/node_modules/@csstools/selector-specificity": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-5.0.0.tgz", + "integrity": "sha512-PCqQV3c4CoVm3kdPhyeZ07VmBRdH2EpMFA/pd9OASpOEC3aXNGoqPDAZ80D0cLpMBxnmk0+yNhGsEx31hq7Gtw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.0.0" + } + }, + "node_modules/@csstools/postcss-is-pseudo-class/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@csstools/postcss-light-dark-function": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@csstools/postcss-light-dark-function/-/postcss-light-dark-function-2.0.7.tgz", + "integrity": "sha512-ZZ0rwlanYKOHekyIPaU+sVm3BEHCe+Ha0/px+bmHe62n0Uc1lL34vbwrLYn6ote8PHlsqzKeTQdIejQCJ05tfw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-logical-float-and-clear": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-logical-float-and-clear/-/postcss-logical-float-and-clear-3.0.0.tgz", + "integrity": "sha512-SEmaHMszwakI2rqKRJgE+8rpotFfne1ZS6bZqBoQIicFyV+xT1UF42eORPxJkVJVrH9C0ctUgwMSn3BLOIZldQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-logical-overflow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-logical-overflow/-/postcss-logical-overflow-2.0.0.tgz", + "integrity": "sha512-spzR1MInxPuXKEX2csMamshR4LRaSZ3UXVaRGjeQxl70ySxOhMpP2252RAFsg8QyyBXBzuVOOdx1+bVO5bPIzA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-logical-overscroll-behavior": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-logical-overscroll-behavior/-/postcss-logical-overscroll-behavior-2.0.0.tgz", + "integrity": "sha512-e/webMjoGOSYfqLunyzByZj5KKe5oyVg/YSbie99VEaSDE2kimFm0q1f6t/6Jo+VVCQ/jbe2Xy+uX+C4xzWs4w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-logical-resize": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-logical-resize/-/postcss-logical-resize-3.0.0.tgz", + "integrity": "sha512-DFbHQOFW/+I+MY4Ycd/QN6Dg4Hcbb50elIJCfnwkRTCX05G11SwViI5BbBlg9iHRl4ytB7pmY5ieAFk3ws7yyg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-logical-viewport-units": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@csstools/postcss-logical-viewport-units/-/postcss-logical-viewport-units-3.0.3.tgz", + "integrity": "sha512-OC1IlG/yoGJdi0Y+7duz/kU/beCwO+Gua01sD6GtOtLi7ByQUpcIqs7UE/xuRPay4cHgOMatWdnDdsIDjnWpPw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-media-minmax": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@csstools/postcss-media-minmax/-/postcss-media-minmax-2.0.5.tgz", + "integrity": "sha512-sdh5i5GToZOIAiwhdntRWv77QDtsxP2r2gXW/WbLSCoLr00KTq/yiF1qlQ5XX2+lmiFa8rATKMcbwl3oXDMNew==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.0", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/media-query-list-parser": "^4.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-media-queries-aspect-ratio-number-values": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/postcss-media-queries-aspect-ratio-number-values/-/postcss-media-queries-aspect-ratio-number-values-3.0.4.tgz", + "integrity": "sha512-AnGjVslHMm5xw9keusQYvjVWvuS7KWK+OJagaG0+m9QnIjZsrysD2kJP/tr/UJIyYtMCtu8OkUd+Rajb4DqtIQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/media-query-list-parser": "^4.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-nested-calc": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-nested-calc/-/postcss-nested-calc-4.0.0.tgz", + "integrity": "sha512-jMYDdqrQQxE7k9+KjstC3NbsmC063n1FTPLCgCRS2/qHUbHM0mNy9pIn4QIiQGs9I/Bg98vMqw7mJXBxa0N88A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-normalize-display-values": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-normalize-display-values/-/postcss-normalize-display-values-4.0.0.tgz", + "integrity": "sha512-HlEoG0IDRoHXzXnkV4in47dzsxdsjdz6+j7MLjaACABX2NfvjFS6XVAnpaDyGesz9gK2SC7MbNwdCHusObKJ9Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-oklab-function": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-oklab-function/-/postcss-oklab-function-4.0.6.tgz", + "integrity": "sha512-Hptoa0uX+XsNacFBCIQKTUBrFKDiplHan42X73EklG6XmQLG7/aIvxoNhvZ7PvOWMt67Pw3bIlUY2nD6p5vL8A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-progressive-custom-properties": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-progressive-custom-properties/-/postcss-progressive-custom-properties-4.0.0.tgz", + "integrity": "sha512-XQPtROaQjomnvLUSy/bALTR5VCtTVUFwYs1SblvYgLSeTo2a/bMNwUwo2piXw5rTv/FEYiy5yPSXBqg9OKUx7Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-random-function": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@csstools/postcss-random-function/-/postcss-random-function-1.0.1.tgz", + "integrity": "sha512-Ab/tF8/RXktQlFwVhiC70UNfpFQRhtE5fQQoP2pO+KCPGLsLdWFiOuHgSRtBOqEshCVAzR4H6o38nhvRZq8deA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-calc": "^2.1.0", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-relative-color-syntax": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@csstools/postcss-relative-color-syntax/-/postcss-relative-color-syntax-3.0.6.tgz", + "integrity": "sha512-yxP618Xb+ji1I624jILaYM62uEmZcmbdmFoZHoaThw896sq0vU39kqTTF+ZNic9XyPtPMvq0vyvbgmHaszq8xg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-scope-pseudo-class": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@csstools/postcss-scope-pseudo-class/-/postcss-scope-pseudo-class-4.0.1.tgz", + "integrity": "sha512-IMi9FwtH6LMNuLea1bjVMQAsUhFxJnyLSgOp/cpv5hrzWmrUYU5fm0EguNDIIOHUqzXode8F/1qkC/tEo/qN8Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-scope-pseudo-class/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@csstools/postcss-sign-functions": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-sign-functions/-/postcss-sign-functions-1.1.0.tgz", + "integrity": "sha512-SLcc20Nujx/kqbSwDmj6oaXgpy3UjFhBy1sfcqPgDkHfOIfUtUVH7OXO+j7BU4v/At5s61N5ZX6shvgPwluhsA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-calc": "^2.1.0", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-stepped-value-functions": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@csstools/postcss-stepped-value-functions/-/postcss-stepped-value-functions-4.0.5.tgz", + "integrity": "sha512-G6SJ6hZJkhxo6UZojVlLo14MohH4J5J7z8CRBrxxUYy9JuZiIqUo5TBYyDGcE0PLdzpg63a7mHSJz3VD+gMwqw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-calc": "^2.1.0", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-text-decoration-shorthand": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@csstools/postcss-text-decoration-shorthand/-/postcss-text-decoration-shorthand-4.0.1.tgz", + "integrity": "sha512-xPZIikbx6jyzWvhms27uugIc0I4ykH4keRvoa3rxX5K7lEhkbd54rjj/dv60qOCTisoS+3bmwJTeyV1VNBrXaw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/color-helpers": "^5.0.1", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-trigonometric-functions": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@csstools/postcss-trigonometric-functions/-/postcss-trigonometric-functions-4.0.5.tgz", + "integrity": "sha512-/YQThYkt5MLvAmVu7zxjhceCYlKrYddK6LEmK5I4ojlS6BmO9u2yO4+xjXzu2+NPYmHSTtP4NFSamBCMmJ1NJA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-calc": "^2.1.0", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/postcss-unset-value": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/postcss-unset-value/-/postcss-unset-value-4.0.0.tgz", + "integrity": "sha512-cBz3tOCI5Fw6NIFEwU3RiwK6mn3nKegjpJuzCndoGq3BZPkUjnsq7uQmIeMNeMbMk7YD2MfKcgCpZwX5jyXqCA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@csstools/utilities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@csstools/utilities/-/utilities-2.0.0.tgz", + "integrity": "sha512-5VdOr0Z71u+Yp3ozOx8T11N703wIFGVRgOWbOZMKgglPJsWA54MRIoMNVMa7shUToIhx5J8vX4sOZgD2XiihiQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/@discoveryjs/json-ext": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", + "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@docsearch/css": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-3.9.0.tgz", + "integrity": "sha512-cQbnVbq0rrBwNAKegIac/t6a8nWoUAn8frnkLFW6YARaRmAQr5/Eoe6Ln2fqkUCZ40KpdrKbpSAmgrkviOxuWA==", + "license": "MIT" + }, + "node_modules/@docsearch/react": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-3.9.0.tgz", + "integrity": "sha512-mb5FOZYZIkRQ6s/NWnM98k879vu5pscWqTLubLFBO87igYYT4VzVazh4h5o/zCvTIZgEt3PvsCOMOswOUo9yHQ==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-core": "1.17.9", + "@algolia/autocomplete-preset-algolia": "1.17.9", + "@docsearch/css": "3.9.0", + "algoliasearch": "^5.14.2" + }, + "peerDependencies": { + "@types/react": ">= 16.8.0 < 20.0.0", + "react": ">= 16.8.0 < 20.0.0", + "react-dom": ">= 16.8.0 < 20.0.0", + "search-insights": ">= 1 < 3" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "search-insights": { + "optional": true + } + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/autocomplete-core": { + "version": "1.17.9", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.17.9.tgz", + "integrity": "sha512-O7BxrpLDPJWWHv/DLA9DRFWs+iY1uOJZkqUwjS5HSZAGcl0hIVCQ97LTLewiZmZ402JYUrun+8NqFP+hCknlbQ==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-plugin-algolia-insights": "1.17.9", + "@algolia/autocomplete-shared": "1.17.9" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/autocomplete-plugin-algolia-insights": { + "version": "1.17.9", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-plugin-algolia-insights/-/autocomplete-plugin-algolia-insights-1.17.9.tgz", + "integrity": "sha512-u1fEHkCbWF92DBeB/KHeMacsjsoI0wFhjZtlCq2ddZbAehshbZST6Hs0Avkc0s+4UyBGbMDnSuXHLuvRWK5iDQ==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-shared": "1.17.9" + }, + "peerDependencies": { + "search-insights": ">= 1 < 3" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/autocomplete-preset-algolia": { + "version": "1.17.9", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.17.9.tgz", + "integrity": "sha512-Na1OuceSJeg8j7ZWn5ssMu/Ax3amtOwk76u4h5J4eK2Nx2KB5qt0Z4cOapCsxot9VcEN11ADV5aUSlQF4RhGjQ==", + "license": "MIT", + "dependencies": { + "@algolia/autocomplete-shared": "1.17.9" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/autocomplete-shared": { + "version": "1.17.9", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.17.9.tgz", + "integrity": "sha512-iDf05JDQ7I0b7JEA/9IektxN/80a2MZ1ToohfmNS3rfeuQnIKI3IJlIafD0xu4StbtQTghx9T3Maa97ytkXenQ==", + "license": "MIT", + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-abtesting": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-abtesting/-/client-abtesting-5.25.0.tgz", + "integrity": "sha512-1pfQulNUYNf1Tk/svbfjfkLBS36zsuph6m+B6gDkPEivFmso/XnRgwDvjAx80WNtiHnmeNjIXdF7Gos8+OLHqQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-analytics": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-5.25.0.tgz", + "integrity": "sha512-AFbG6VDJX/o2vDd9hqncj1B6B4Tulk61mY0pzTtzKClyTDlNP0xaUiEKhl6E7KO9I/x0FJF5tDCm0Hn6v5x18A==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-common": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.25.0.tgz", + "integrity": "sha512-il1zS/+Rc6la6RaCdSZ2YbJnkQC6W1wiBO8+SH+DE6CPMWBU6iDVzH0sCKSAtMWl9WBxoN6MhNjGBnCv9Yy2bA==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-insights": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-insights/-/client-insights-5.25.0.tgz", + "integrity": "sha512-blbjrUH1siZNfyCGeq0iLQu00w3a4fBXm0WRIM0V8alcAPo7rWjLbMJMrfBtzL9X5ic6wgxVpDADXduGtdrnkw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-personalization": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-5.25.0.tgz", + "integrity": "sha512-aywoEuu1NxChBcHZ1pWaat0Plw7A8jDMwjgRJ00Mcl7wGlwuPt5dJ/LTNcg3McsEUbs2MBNmw0ignXBw9Tbgow==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-query-suggestions": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-query-suggestions/-/client-query-suggestions-5.25.0.tgz", + "integrity": "sha512-a/W2z6XWKjKjIW1QQQV8PTTj1TXtaKx79uR3NGBdBdGvVdt24KzGAaN7sCr5oP8DW4D3cJt44wp2OY/fZcPAVA==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/client-search": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.25.0.tgz", + "integrity": "sha512-9rUYcMIBOrCtYiLX49djyzxqdK9Dya/6Z/8sebPn94BekT+KLOpaZCuc6s0Fpfq7nx5J6YY5LIVFQrtioK9u0g==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/ingestion": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@algolia/ingestion/-/ingestion-1.25.0.tgz", + "integrity": "sha512-jJeH/Hk+k17Vkokf02lkfYE4A+EJX+UgnMhTLR/Mb+d1ya5WhE+po8p5a/Nxb6lo9OLCRl6w3Hmk1TX1e9gVbQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/monitoring": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@algolia/monitoring/-/monitoring-1.25.0.tgz", + "integrity": "sha512-Ls3i1AehJ0C6xaHe7kK9vPmzImOn5zBg7Kzj8tRYIcmCWVyuuFwCIsbuIIz/qzUf1FPSWmw0TZrGeTumk2fqXg==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/recommend": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/recommend/-/recommend-5.25.0.tgz", + "integrity": "sha512-79sMdHpiRLXVxSjgw7Pt4R1aNUHxFLHiaTDnN2MQjHwJ1+o3wSseb55T9VXU4kqy3m7TUme3pyRhLk5ip/S4Mw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/requester-browser-xhr": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.25.0.tgz", + "integrity": "sha512-JLaF23p1SOPBmfEqozUAgKHQrGl3z/Z5RHbggBu6s07QqXXcazEsub5VLonCxGVqTv6a61AAPr8J1G5HgGGjEw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/requester-fetch": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-fetch/-/requester-fetch-5.25.0.tgz", + "integrity": "sha512-rtzXwqzFi1edkOF6sXxq+HhmRKDy7tz84u0o5t1fXwz0cwx+cjpmxu/6OQKTdOJFS92JUYHsG51Iunie7xbqfQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/@algolia/requester-node-http": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.25.0.tgz", + "integrity": "sha512-ZO0UKvDyEFvyeJQX0gmZDQEvhLZ2X10K+ps6hViMo1HgE2V8em00SwNsQ+7E/52a+YiBkVWX61pJJJE44juDMQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docsearch/react/node_modules/algoliasearch": { + "version": "5.25.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.25.0.tgz", + "integrity": "sha512-n73BVorL4HIwKlfJKb4SEzAYkR3Buwfwbh+MYxg2mloFph2fFGV58E90QTzdbfzWrLn4HE5Czx/WTjI8fcHaMg==", + "license": "MIT", + "dependencies": { + "@algolia/client-abtesting": "5.25.0", + "@algolia/client-analytics": "5.25.0", + "@algolia/client-common": "5.25.0", + "@algolia/client-insights": "5.25.0", + "@algolia/client-personalization": "5.25.0", + "@algolia/client-query-suggestions": "5.25.0", + "@algolia/client-search": "5.25.0", + "@algolia/ingestion": "1.25.0", + "@algolia/monitoring": "1.25.0", + "@algolia/recommend": "5.25.0", + "@algolia/requester-browser-xhr": "5.25.0", + "@algolia/requester-fetch": "5.25.0", + "@algolia/requester-node-http": "5.25.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/babel": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/babel/-/babel-3.8.0.tgz", + "integrity": "sha512-9EJwSgS6TgB8IzGk1L8XddJLhZod8fXT4ULYMx6SKqyCBqCFpVCEjR/hNXXhnmtVM2irDuzYoVLGWv7srG/VOA==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.9", + "@babel/generator": "^7.25.9", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-transform-runtime": "^7.25.9", + "@babel/preset-env": "^7.25.9", + "@babel/preset-react": "^7.25.9", + "@babel/preset-typescript": "^7.25.9", + "@babel/runtime": "^7.25.9", + "@babel/runtime-corejs3": "^7.25.9", + "@babel/traverse": "^7.25.9", + "@docusaurus/logger": "3.8.0", + "@docusaurus/utils": "3.8.0", + "babel-plugin-dynamic-import-node": "^2.3.3", + "fs-extra": "^11.1.1", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/bundler": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/bundler/-/bundler-3.8.0.tgz", + "integrity": "sha512-Rq4Z/MSeAHjVzBLirLeMcjLIAQy92pF1OI+2rmt18fSlMARfTGLWRE8Vb+ljQPTOSfJxwDYSzsK6i7XloD2rNA==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.9", + "@docusaurus/babel": "3.8.0", + "@docusaurus/cssnano-preset": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "babel-loader": "^9.2.1", + "clean-css": "^5.3.2", + "copy-webpack-plugin": "^11.0.0", + "css-loader": "^6.8.1", + "css-minimizer-webpack-plugin": "^5.0.1", + "cssnano": "^6.1.2", + "file-loader": "^6.2.0", + "html-minifier-terser": "^7.2.0", + "mini-css-extract-plugin": "^2.9.1", + "null-loader": "^4.0.1", + "postcss": "^8.4.26", + "postcss-loader": "^7.3.3", + "postcss-preset-env": "^10.1.0", + "terser-webpack-plugin": "^5.3.9", + "tslib": "^2.6.0", + "url-loader": "^4.1.1", + "webpack": "^5.95.0", + "webpackbar": "^6.0.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "@docusaurus/faster": "*" + }, + "peerDependenciesMeta": { + "@docusaurus/faster": { + "optional": true + } + } + }, + "node_modules/@docusaurus/core": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/core/-/core-3.8.0.tgz", + "integrity": "sha512-c7u6zFELmSGPEP9WSubhVDjgnpiHgDqMh1qVdCB7rTflh4Jx0msTYmMiO91Ez0KtHj4sIsDsASnjwfJ2IZp3Vw==", + "license": "MIT", + "dependencies": { + "@docusaurus/babel": "3.8.0", + "@docusaurus/bundler": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/mdx-loader": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "boxen": "^6.2.1", + "chalk": "^4.1.2", + "chokidar": "^3.5.3", + "cli-table3": "^0.6.3", + "combine-promises": "^1.1.0", + "commander": "^5.1.0", + "core-js": "^3.31.1", + "detect-port": "^1.5.1", + "escape-html": "^1.0.3", + "eta": "^2.2.0", + "eval": "^0.1.8", + "execa": "5.1.1", + "fs-extra": "^11.1.1", + "html-tags": "^3.3.1", + "html-webpack-plugin": "^5.6.0", + "leven": "^3.1.0", + "lodash": "^4.17.21", + "open": "^8.4.0", + "p-map": "^4.0.0", + "prompts": "^2.4.2", + "react-helmet-async": "npm:@slorber/react-helmet-async@1.3.0", + "react-loadable": "npm:@docusaurus/react-loadable@6.0.0", + "react-loadable-ssr-addon-v5-slorber": "^1.0.1", + "react-router": "^5.3.4", + "react-router-config": "^5.1.1", + "react-router-dom": "^5.3.4", + "semver": "^7.5.4", + "serve-handler": "^6.1.6", + "tinypool": "^1.0.2", + "tslib": "^2.6.0", + "update-notifier": "^6.0.2", + "webpack": "^5.95.0", + "webpack-bundle-analyzer": "^4.10.2", + "webpack-dev-server": "^4.15.2", + "webpack-merge": "^6.0.1" + }, + "bin": { + "docusaurus": "bin/docusaurus.mjs" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "@mdx-js/react": "^3.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/cssnano-preset": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/cssnano-preset/-/cssnano-preset-3.8.0.tgz", + "integrity": "sha512-UJ4hAS2T0R4WNy+phwVff2Q0L5+RXW9cwlH6AEphHR5qw3m/yacfWcSK7ort2pMMbDn8uGrD38BTm4oLkuuNoQ==", + "license": "MIT", + "dependencies": { + "cssnano-preset-advanced": "^6.1.2", + "postcss": "^8.4.38", + "postcss-sort-media-queries": "^5.2.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/logger": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/logger/-/logger-3.8.0.tgz", + "integrity": "sha512-7eEMaFIam5Q+v8XwGqF/n0ZoCld4hV4eCCgQkfcN9Mq5inoZa6PHHW9Wu6lmgzoK5Kx3keEeABcO2SxwraoPDQ==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/mdx-loader": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/mdx-loader/-/mdx-loader-3.8.0.tgz", + "integrity": "sha512-mDPSzssRnpjSdCGuv7z2EIAnPS1MHuZGTaRLwPn4oQwszu4afjWZ/60sfKjTnjBjI8Vl4OgJl2vMmfmiNDX4Ng==", + "license": "MIT", + "dependencies": { + "@docusaurus/logger": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "@mdx-js/mdx": "^3.0.0", + "@slorber/remark-comment": "^1.0.0", + "escape-html": "^1.0.3", + "estree-util-value-to-estree": "^3.0.1", + "file-loader": "^6.2.0", + "fs-extra": "^11.1.1", + "image-size": "^2.0.2", + "mdast-util-mdx": "^3.0.0", + "mdast-util-to-string": "^4.0.0", + "rehype-raw": "^7.0.0", + "remark-directive": "^3.0.0", + "remark-emoji": "^4.0.0", + "remark-frontmatter": "^5.0.0", + "remark-gfm": "^4.0.0", + "stringify-object": "^3.3.0", + "tslib": "^2.6.0", + "unified": "^11.0.3", + "unist-util-visit": "^5.0.0", + "url-loader": "^4.1.1", + "vfile": "^6.0.1", + "webpack": "^5.88.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/module-type-aliases": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/module-type-aliases/-/module-type-aliases-3.8.0.tgz", + "integrity": "sha512-/uMb4Ipt5J/QnD13MpnoC/A4EYAe6DKNWqTWLlGrqsPJwJv73vSwkA25xnYunwfqWk0FlUQfGv/Swdh5eCCg7g==", + "license": "MIT", + "dependencies": { + "@docusaurus/types": "3.8.0", + "@types/history": "^4.7.11", + "@types/react": "*", + "@types/react-router-config": "*", + "@types/react-router-dom": "*", + "react-helmet-async": "npm:@slorber/react-helmet-async@1.3.0", + "react-loadable": "npm:@docusaurus/react-loadable@6.0.0" + }, + "peerDependencies": { + "react": "*", + "react-dom": "*" + } + }, + "node_modules/@docusaurus/plugin-content-blog": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.8.0.tgz", + "integrity": "sha512-0SlOTd9R55WEr1GgIXu+hhTT0hzARYx3zIScA5IzpdekZQesI/hKEa5LPHBd415fLkWMjdD59TaW/3qQKpJ0Lg==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/mdx-loader": "3.8.0", + "@docusaurus/theme-common": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "cheerio": "1.0.0-rc.12", + "feed": "^4.2.2", + "fs-extra": "^11.1.1", + "lodash": "^4.17.21", + "schema-dts": "^1.1.2", + "srcset": "^4.0.0", + "tslib": "^2.6.0", + "unist-util-visit": "^5.0.0", + "utility-types": "^3.10.0", + "webpack": "^5.88.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "@docusaurus/plugin-content-docs": "*", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-content-docs": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.8.0.tgz", + "integrity": "sha512-fRDMFLbUN6eVRXcjP8s3Y7HpAt9pzPYh1F/7KKXOCxvJhjjCtbon4VJW0WndEPInVz4t8QUXn5QZkU2tGVCE2g==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/mdx-loader": "3.8.0", + "@docusaurus/module-type-aliases": "3.8.0", + "@docusaurus/theme-common": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "@types/react-router-config": "^5.0.7", + "combine-promises": "^1.1.0", + "fs-extra": "^11.1.1", + "js-yaml": "^4.1.0", + "lodash": "^4.17.21", + "schema-dts": "^1.1.2", + "tslib": "^2.6.0", + "utility-types": "^3.10.0", + "webpack": "^5.88.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-content-pages": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-pages/-/plugin-content-pages-3.8.0.tgz", + "integrity": "sha512-39EDx2y1GA0Pxfion5tQZLNJxL4gq6susd1xzetVBjVIQtwpCdyloOfQBAgX0FylqQxfJrYqL0DIUuq7rd7uBw==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/mdx-loader": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "fs-extra": "^11.1.1", + "tslib": "^2.6.0", + "webpack": "^5.88.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-css-cascade-layers": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-css-cascade-layers/-/plugin-css-cascade-layers-3.8.0.tgz", + "integrity": "sha512-/VBTNymPIxQB8oA3ZQ4GFFRYdH4ZxDRRBECxyjRyv486mfUPXfcdk+im4S5mKWa6EK2JzBz95IH/Wu0qQgJ5yQ==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/plugin-debug": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-debug/-/plugin-debug-3.8.0.tgz", + "integrity": "sha512-teonJvJsDB9o2OnG6ifbhblg/PXzZvpUKHFgD8dOL1UJ58u0lk8o0ZOkvaYEBa9nDgqzoWrRk9w+e3qaG2mOhQ==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "fs-extra": "^11.1.1", + "react-json-view-lite": "^2.3.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-google-analytics": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-analytics/-/plugin-google-analytics-3.8.0.tgz", + "integrity": "sha512-aKKa7Q8+3xRSRESipNvlFgNp3FNPELKhuo48Cg/svQbGNwidSHbZT03JqbW4cBaQnyyVchO1ttk+kJ5VC9Gx0w==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-google-gtag": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-gtag/-/plugin-google-gtag-3.8.0.tgz", + "integrity": "sha512-ugQYMGF4BjbAW/JIBtVcp+9eZEgT9HRdvdcDudl5rywNPBA0lct+lXMG3r17s02rrhInMpjMahN3Yc9Cb3H5/g==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "@types/gtag.js": "^0.0.12", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-google-tag-manager": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-google-tag-manager/-/plugin-google-tag-manager-3.8.0.tgz", + "integrity": "sha512-9juRWxbwZD3SV02Jd9QB6yeN7eu+7T4zB0bvJLcVQwi+am51wAxn2CwbdL0YCCX+9OfiXbADE8D8Q65Hbopu/w==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-sitemap": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-sitemap/-/plugin-sitemap-3.8.0.tgz", + "integrity": "sha512-fGpOIyJvNiuAb90nSJ2Gfy/hUOaDu6826e5w5UxPmbpCIc7KlBHNAZ5g4L4ZuHhc4hdfq4mzVBsQSnne+8Ze1g==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "fs-extra": "^11.1.1", + "sitemap": "^7.1.1", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/plugin-svgr": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/plugin-svgr/-/plugin-svgr-3.8.0.tgz", + "integrity": "sha512-kEDyry+4OMz6BWLG/lEqrNsL/w818bywK70N1gytViw4m9iAmoxCUT7Ri9Dgs7xUdzCHJ3OujolEmD88Wy44OA==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "@svgr/core": "8.1.0", + "@svgr/webpack": "^8.1.0", + "tslib": "^2.6.0", + "webpack": "^5.88.1" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/preset-classic": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/preset-classic/-/preset-classic-3.8.0.tgz", + "integrity": "sha512-qOu6tQDOWv+rpTlKu+eJATCJVGnABpRCPuqf7LbEaQ1mNY//N/P8cHQwkpAU+aweQfarcZ0XfwCqRHJfjeSV/g==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/plugin-content-blog": "3.8.0", + "@docusaurus/plugin-content-docs": "3.8.0", + "@docusaurus/plugin-content-pages": "3.8.0", + "@docusaurus/plugin-css-cascade-layers": "3.8.0", + "@docusaurus/plugin-debug": "3.8.0", + "@docusaurus/plugin-google-analytics": "3.8.0", + "@docusaurus/plugin-google-gtag": "3.8.0", + "@docusaurus/plugin-google-tag-manager": "3.8.0", + "@docusaurus/plugin-sitemap": "3.8.0", + "@docusaurus/plugin-svgr": "3.8.0", + "@docusaurus/theme-classic": "3.8.0", + "@docusaurus/theme-common": "3.8.0", + "@docusaurus/theme-search-algolia": "3.8.0", + "@docusaurus/types": "3.8.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/theme-classic": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/theme-classic/-/theme-classic-3.8.0.tgz", + "integrity": "sha512-nQWFiD5ZjoT76OaELt2n33P3WVuuCz8Dt5KFRP2fCBo2r9JCLsp2GJjZpnaG24LZ5/arRjv4VqWKgpK0/YLt7g==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/mdx-loader": "3.8.0", + "@docusaurus/module-type-aliases": "3.8.0", + "@docusaurus/plugin-content-blog": "3.8.0", + "@docusaurus/plugin-content-docs": "3.8.0", + "@docusaurus/plugin-content-pages": "3.8.0", + "@docusaurus/theme-common": "3.8.0", + "@docusaurus/theme-translations": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "@mdx-js/react": "^3.0.0", + "clsx": "^2.0.0", + "copy-text-to-clipboard": "^3.2.0", + "infima": "0.2.0-alpha.45", + "lodash": "^4.17.21", + "nprogress": "^0.2.0", + "postcss": "^8.4.26", + "prism-react-renderer": "^2.3.0", + "prismjs": "^1.29.0", + "react-router-dom": "^5.3.4", + "rtlcss": "^4.1.0", + "tslib": "^2.6.0", + "utility-types": "^3.10.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/theme-common": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/theme-common/-/theme-common-3.8.0.tgz", + "integrity": "sha512-YqV2vAWpXGLA+A3PMLrOMtqgTHJLDcT+1Caa6RF7N4/IWgrevy5diY8oIHFkXR/eybjcrFFjUPrHif8gSGs3Tw==", + "license": "MIT", + "dependencies": { + "@docusaurus/mdx-loader": "3.8.0", + "@docusaurus/module-type-aliases": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "@types/history": "^4.7.11", + "@types/react": "*", + "@types/react-router-config": "*", + "clsx": "^2.0.0", + "parse-numeric-range": "^1.3.0", + "prism-react-renderer": "^2.3.0", + "tslib": "^2.6.0", + "utility-types": "^3.10.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "@docusaurus/plugin-content-docs": "*", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/theme-mermaid": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/theme-mermaid/-/theme-mermaid-3.8.0.tgz", + "integrity": "sha512-ou0NJM37p4xrVuFaZp8qFe5Z/qBq9LuyRTP4KKRa0u2J3zC4f3saBJDgc56FyvvN1OsmU0189KGEPUjTr6hFxg==", + "license": "MIT", + "dependencies": { + "@docusaurus/core": "3.8.0", + "@docusaurus/module-type-aliases": "3.8.0", + "@docusaurus/theme-common": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "mermaid": ">=11.6.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/theme-search-algolia/-/theme-search-algolia-3.8.0.tgz", + "integrity": "sha512-GBZ5UOcPgiu6nUw153+0+PNWvFKweSnvKIL6Rp04H9olKb475jfKjAwCCtju5D2xs5qXHvCMvzWOg5o9f6DtuQ==", + "license": "MIT", + "dependencies": { + "@docsearch/react": "^3.9.0", + "@docusaurus/core": "3.8.0", + "@docusaurus/logger": "3.8.0", + "@docusaurus/plugin-content-docs": "3.8.0", + "@docusaurus/theme-common": "3.8.0", + "@docusaurus/theme-translations": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-validation": "3.8.0", + "algoliasearch": "^5.17.1", + "algoliasearch-helper": "^3.22.6", + "clsx": "^2.0.0", + "eta": "^2.2.0", + "fs-extra": "^11.1.1", + "lodash": "^4.17.21", + "tslib": "^2.6.0", + "utility-types": "^3.10.0" + }, + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/client-analytics": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-5.19.0.tgz", + "integrity": "sha512-CDW4RwnCHzU10upPJqS6N6YwDpDHno7w6/qXT9KPbPbt8szIIzCHrva4O9KIfx1OhdsHzfGSI5hMAiOOYl4DEQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/client-common": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.19.0.tgz", + "integrity": "sha512-2ERRbICHXvtj5kfFpY5r8qu9pJII/NAHsdgUXnUitQFwPdPL7wXiupcvZJC7DSntOnE8AE0lM7oDsPhrJfj5nQ==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/client-personalization": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-5.19.0.tgz", + "integrity": "sha512-B9eoce/fk8NLboGje+pMr72pw+PV7c5Z01On477heTZ7jkxoZ4X92dobeGuEQop61cJ93Gaevd1of4mBr4hu2A==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/client-search": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.19.0.tgz", + "integrity": "sha512-Ctg3xXD/1VtcwmkulR5+cKGOMj4r0wC49Y/KZdGQcqpydKn+e86F6l3tb3utLJQVq4lpEJud6kdRykFgcNsp8Q==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/recommend": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/recommend/-/recommend-5.19.0.tgz", + "integrity": "sha512-PbgrMTbUPlmwfJsxjFhal4XqZO2kpBNRjemLVTkUiti4w/+kzcYO4Hg5zaBgVqPwvFDNQ8JS4SS3TBBem88u+g==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/requester-browser-xhr": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.19.0.tgz", + "integrity": "sha512-GfnhnQBT23mW/VMNs7m1qyEyZzhZz093aY2x8p0era96MMyNv8+FxGek5pjVX0b57tmSCZPf4EqNCpkGcGsmbw==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/@algolia/requester-node-http": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.19.0.tgz", + "integrity": "sha512-p6t8ue0XZNjcRiqNkb5QAM0qQRAKsCiebZ6n9JjWA+p8fWf8BvnhO55y2fO28g3GW0Imj7PrAuyBuxq8aDVQwQ==", + "license": "MIT", + "dependencies": { + "@algolia/client-common": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-search-algolia/node_modules/algoliasearch": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.19.0.tgz", + "integrity": "sha512-zrLtGhC63z3sVLDDKGW+SlCRN9eJHFTgdEmoAOpsVh6wgGL1GgTTDou7tpCBjevzgIvi3AIyDAQO3Xjbg5eqZg==", + "license": "MIT", + "dependencies": { + "@algolia/client-abtesting": "5.19.0", + "@algolia/client-analytics": "5.19.0", + "@algolia/client-common": "5.19.0", + "@algolia/client-insights": "5.19.0", + "@algolia/client-personalization": "5.19.0", + "@algolia/client-query-suggestions": "5.19.0", + "@algolia/client-search": "5.19.0", + "@algolia/ingestion": "1.19.0", + "@algolia/monitoring": "1.19.0", + "@algolia/recommend": "5.19.0", + "@algolia/requester-browser-xhr": "5.19.0", + "@algolia/requester-fetch": "5.19.0", + "@algolia/requester-node-http": "5.19.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@docusaurus/theme-translations": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/theme-translations/-/theme-translations-3.8.0.tgz", + "integrity": "sha512-1DTy/snHicgkCkryWq54fZvsAglTdjTx4qjOXgqnXJ+DIty1B+aPQrAVUu8LiM+6BiILfmNxYsxhKTj+BS3PZg==", + "license": "MIT", + "dependencies": { + "fs-extra": "^11.1.1", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/types": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/types/-/types-3.8.0.tgz", + "integrity": "sha512-RDEClpwNxZq02c+JlaKLWoS13qwWhjcNsi2wG1UpzmEnuti/z1Wx4SGpqbUqRPNSd8QWWePR8Cb7DvG0VN/TtA==", + "license": "MIT", + "dependencies": { + "@mdx-js/mdx": "^3.0.0", + "@types/history": "^4.7.11", + "@types/react": "*", + "commander": "^5.1.0", + "joi": "^17.9.2", + "react-helmet-async": "npm:@slorber/react-helmet-async@1.3.0", + "utility-types": "^3.10.0", + "webpack": "^5.95.0", + "webpack-merge": "^5.9.0" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/@docusaurus/types/node_modules/webpack-merge": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.10.0.tgz", + "integrity": "sha512-+4zXKdx7UnO+1jaN4l2lHVD+mFvnlZQP/6ljaJVb4SZiwIKeUnrT5l0gkT8z+n4hKpC+jpOv6O9R+gLtag7pSA==", + "license": "MIT", + "dependencies": { + "clone-deep": "^4.0.1", + "flat": "^5.0.2", + "wildcard": "^2.0.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@docusaurus/utils": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/utils/-/utils-3.8.0.tgz", + "integrity": "sha512-2wvtG28ALCN/A1WCSLxPASFBFzXCnP0YKCAFIPcvEb6imNu1wg7ni/Svcp71b3Z2FaOFFIv4Hq+j4gD7gA0yfQ==", + "license": "MIT", + "dependencies": { + "@docusaurus/logger": "3.8.0", + "@docusaurus/types": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "escape-string-regexp": "^4.0.0", + "execa": "5.1.1", + "file-loader": "^6.2.0", + "fs-extra": "^11.1.1", + "github-slugger": "^1.5.0", + "globby": "^11.1.0", + "gray-matter": "^4.0.3", + "jiti": "^1.20.0", + "js-yaml": "^4.1.0", + "lodash": "^4.17.21", + "micromatch": "^4.0.5", + "p-queue": "^6.6.2", + "prompts": "^2.4.2", + "resolve-pathname": "^3.0.0", + "tslib": "^2.6.0", + "url-loader": "^4.1.1", + "utility-types": "^3.10.0", + "webpack": "^5.88.1" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/utils-common": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/utils-common/-/utils-common-3.8.0.tgz", + "integrity": "sha512-3TGF+wVTGgQ3pAc9+5jVchES4uXUAhAt9pwv7uws4mVOxL4alvU3ue/EZ+R4XuGk94pDy7CNXjRXpPjlfZXQfw==", + "license": "MIT", + "dependencies": { + "@docusaurus/types": "3.8.0", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@docusaurus/utils-validation": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@docusaurus/utils-validation/-/utils-validation-3.8.0.tgz", + "integrity": "sha512-MrnEbkigr54HkdFeg8e4FKc4EF+E9dlVwsY3XQZsNkbv3MKZnbHQ5LsNJDIKDROFe8PBf5C4qCAg5TPBpsjrjg==", + "license": "MIT", + "dependencies": { + "@docusaurus/logger": "3.8.0", + "@docusaurus/utils": "3.8.0", + "@docusaurus/utils-common": "3.8.0", + "fs-extra": "^11.2.0", + "joi": "^17.9.2", + "js-yaml": "^4.1.0", + "lodash": "^4.17.21", + "tslib": "^2.6.0" + }, + "engines": { + "node": ">=18.0" + } + }, + "node_modules/@hapi/hoek": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@hapi/topo": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz", + "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^9.0.0" + } + }, + "node_modules/@iconify/types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", + "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", + "license": "MIT" + }, + "node_modules/@iconify/utils": { + "version": "2.1.33", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-2.1.33.tgz", + "integrity": "sha512-jP9h6v/g0BIZx0p7XGJJVtkVnydtbgTgt9mVNcGDYwaa7UhdHdI9dvoq+gKj9sijMSJKxUPEG2JyjsgXjxL7Kw==", + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^0.4.0", + "@antfu/utils": "^0.7.10", + "@iconify/types": "^2.0.0", + "debug": "^4.3.6", + "kolorist": "^1.8.0", + "local-pkg": "^0.5.0", + "mlly": "^1.7.1" + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", + "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", + "license": "MIT" + }, + "node_modules/@mdx-js/mdx": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-3.1.0.tgz", + "integrity": "sha512-/QxEhPAvGwbQmy1Px8F899L5Uc2KZ6JtXwlCgJmjSTBedwOZkByYcBG4GceIGPXRDsmfxhHazuS+hlOShRLeDw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdx": "^2.0.0", + "collapse-white-space": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-util-scope": "^1.0.0", + "estree-walker": "^3.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "markdown-extensions": "^2.0.0", + "recma-build-jsx": "^1.0.0", + "recma-jsx": "^1.0.0", + "recma-stringify": "^1.0.0", + "rehype-recma": "^1.0.0", + "remark-mdx": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "source-map": "^0.7.0", + "unified": "^11.0.0", + "unist-util-position-from-estree": "^2.0.0", + "unist-util-stringify-position": "^4.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@mdx-js/react": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-3.1.0.tgz", + "integrity": "sha512-QjHtSaoameoalGnKDT3FoIl4+9RwyTmo9ZJGBdLOks/YOiWHoRDI3PUwEzOE7kEmGcV3AFcp9K6dYu9rEuKLAQ==", + "license": "MIT", + "dependencies": { + "@types/mdx": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=16", + "react": ">=16" + } + }, + "node_modules/@mermaid-js/parser": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-0.4.0.tgz", + "integrity": "sha512-wla8XOWvQAwuqy+gxiZqY+c7FokraOTHRWMsbB4AgRx9Sy7zKslNyejy7E+a77qHfey5GXw/ik3IXv/NHMJgaA==", + "license": "MIT", + "dependencies": { + "langium": "3.3.1" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@pnpm/config.env-replace": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", + "integrity": "sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==", + "license": "MIT", + "engines": { + "node": ">=12.22.0" + } + }, + "node_modules/@pnpm/network.ca-file": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@pnpm/network.ca-file/-/network.ca-file-1.0.2.tgz", + "integrity": "sha512-YcPQ8a0jwYU9bTdJDpXjMi7Brhkr1mXsXrUJvjqM2mQDgkRiz8jFaQGOdaLxgjtUfQgZhKy/O3cG/YwmgKaxLA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "4.2.10" + }, + "engines": { + "node": ">=12.22.0" + } + }, + "node_modules/@pnpm/network.ca-file/node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "license": "ISC" + }, + "node_modules/@pnpm/npm-conf": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@pnpm/npm-conf/-/npm-conf-2.3.1.tgz", + "integrity": "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw==", + "license": "MIT", + "dependencies": { + "@pnpm/config.env-replace": "^1.1.0", + "@pnpm/network.ca-file": "^1.0.1", + "config-chain": "^1.1.11" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@polka/url": { + "version": "1.0.0-next.28", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", + "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==", + "license": "MIT" + }, + "node_modules/@sideway/address": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", + "integrity": "sha512-IqO/DUQHUkPeixNQ8n0JA6102hT9CmaljNTPmQ1u8MEhBo/R4Q8eKLN/vGZxuebwOroDB4cbpjheD4+/sKFK4Q==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^9.0.0" + } + }, + "node_modules/@sideway/formula": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz", + "integrity": "sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==", + "license": "BSD-3-Clause" + }, + "node_modules/@sideway/pinpoint": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz", + "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@slorber/remark-comment": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@slorber/remark-comment/-/remark-comment-1.0.0.tgz", + "integrity": "sha512-RCE24n7jsOj1M0UPvIQCHTe7fI0sFL4S2nwKVWwHyVr/wI/H8GosgsJGyhnsZoGFnD/P2hLf1mSbrrgSLN93NA==", + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.1.0", + "micromark-util-symbol": "^1.0.1" + } + }, + "node_modules/@svgr/babel-plugin-add-jsx-attribute": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-8.0.0.tgz", + "integrity": "sha512-b9MIk7yhdS1pMCZM8VeNfUlSKVRhsHZNMl5O9SfaX0l0t5wjdgu4IDzGB8bpnGBBOjGST3rRFVsaaEtI4W6f7g==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-remove-jsx-attribute": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-8.0.0.tgz", + "integrity": "sha512-BcCkm/STipKvbCl6b7QFrMh/vx00vIP63k2eM66MfHJzPr6O2U0jYEViXkHJWqXqQYjdeA9cuCl5KWmlwjDvbA==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-remove-jsx-empty-expression": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-8.0.0.tgz", + "integrity": "sha512-5BcGCBfBxB5+XSDSWnhTThfI9jcO5f0Ai2V24gZpG+wXF14BzwxxdDb4g6trdOux0rhibGs385BeFMSmxtS3uA==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-replace-jsx-attribute-value": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-8.0.0.tgz", + "integrity": "sha512-KVQ+PtIjb1BuYT3ht8M5KbzWBhdAjjUPdlMtpuw/VjT8coTrItWX6Qafl9+ji831JaJcu6PJNKCV0bp01lBNzQ==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-svg-dynamic-title": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-8.0.0.tgz", + "integrity": "sha512-omNiKqwjNmOQJ2v6ge4SErBbkooV2aAWwaPFs2vUY7p7GhVkzRkJ00kILXQvRhA6miHnNpXv7MRnnSjdRjK8og==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-svg-em-dimensions": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-8.0.0.tgz", + "integrity": "sha512-mURHYnu6Iw3UBTbhGwE/vsngtCIbHE43xCRK7kCw4t01xyGqb2Pd+WXekRRoFOBIY29ZoOhUCTEweDMdrjfi9g==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-transform-react-native-svg": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-8.1.0.tgz", + "integrity": "sha512-Tx8T58CHo+7nwJ+EhUwx3LfdNSG9R2OKfaIXXs5soiy5HtgoAEkDay9LIimLOcG8dJQH1wPZp/cnAv6S9CrR1Q==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-plugin-transform-svg-component": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-8.0.0.tgz", + "integrity": "sha512-DFx8xa3cZXTdb/k3kfPeaixecQLgKh5NVBMwD0AQxOzcZawK4oo1Jh9LbrcACUivsCA7TLG8eeWgrDXjTMhRmw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/babel-preset": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-preset/-/babel-preset-8.1.0.tgz", + "integrity": "sha512-7EYDbHE7MxHpv4sxvnVPngw5fuR6pw79SkcrILHJ/iMpuKySNCl5W1qcwPEpU+LgyRXOaAFgH0KhwD18wwg6ug==", + "license": "MIT", + "dependencies": { + "@svgr/babel-plugin-add-jsx-attribute": "8.0.0", + "@svgr/babel-plugin-remove-jsx-attribute": "8.0.0", + "@svgr/babel-plugin-remove-jsx-empty-expression": "8.0.0", + "@svgr/babel-plugin-replace-jsx-attribute-value": "8.0.0", + "@svgr/babel-plugin-svg-dynamic-title": "8.0.0", + "@svgr/babel-plugin-svg-em-dimensions": "8.0.0", + "@svgr/babel-plugin-transform-react-native-svg": "8.1.0", + "@svgr/babel-plugin-transform-svg-component": "8.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@svgr/core": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz", + "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.21.3", + "@svgr/babel-preset": "8.1.0", + "camelcase": "^6.2.0", + "cosmiconfig": "^8.1.3", + "snake-case": "^3.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + } + }, + "node_modules/@svgr/hast-util-to-babel-ast": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-8.0.0.tgz", + "integrity": "sha512-EbDKwO9GpfWP4jN9sGdYwPBU0kdomaPIL2Eu4YwmgP+sJeXT+L7bMwJUBnhzfH8Q2qMBqZ4fJwpCyYsAN3mt2Q==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.21.3", + "entities": "^4.4.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + } + }, + "node_modules/@svgr/plugin-jsx": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/plugin-jsx/-/plugin-jsx-8.1.0.tgz", + "integrity": "sha512-0xiIyBsLlr8quN+WyuxooNW9RJ0Dpr8uOnH/xrCVO8GLUcwHISwj1AG0k+LFzteTkAA0GbX0kj9q6Dk70PTiPA==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.21.3", + "@svgr/babel-preset": "8.1.0", + "@svgr/hast-util-to-babel-ast": "8.0.0", + "svg-parser": "^2.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@svgr/core": "*" + } + }, + "node_modules/@svgr/plugin-svgo": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/plugin-svgo/-/plugin-svgo-8.1.0.tgz", + "integrity": "sha512-Ywtl837OGO9pTLIN/onoWLmDQ4zFUycI1g76vuKGEz6evR/ZTJlJuz3G/fIkb6OVBJ2g0o6CGJzaEjfmEo3AHA==", + "license": "MIT", + "dependencies": { + "cosmiconfig": "^8.1.3", + "deepmerge": "^4.3.1", + "svgo": "^3.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + }, + "peerDependencies": { + "@svgr/core": "*" + } + }, + "node_modules/@svgr/webpack": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@svgr/webpack/-/webpack-8.1.0.tgz", + "integrity": "sha512-LnhVjMWyMQV9ZmeEy26maJk+8HTIbd59cH4F2MJ439k9DqejRisfFNGAPvRYlKETuh9LrImlS8aKsBgKjMA8WA==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.21.3", + "@babel/plugin-transform-react-constant-elements": "^7.21.3", + "@babel/preset-env": "^7.20.2", + "@babel/preset-react": "^7.18.6", + "@babel/preset-typescript": "^7.21.0", + "@svgr/core": "8.1.0", + "@svgr/plugin-jsx": "8.1.0", + "@svgr/plugin-svgo": "8.1.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/gregberge" + } + }, + "node_modules/@szmarczak/http-timer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz", + "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==", + "license": "MIT", + "dependencies": { + "defer-to-connect": "^2.0.1" + }, + "engines": { + "node": ">=14.16" + } + }, + "node_modules/@trysound/sax": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", + "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", + "license": "ISC", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/@types/acorn": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz", + "integrity": "sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bonjour": { + "version": "3.5.13", + "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.13.tgz", + "integrity": "sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect-history-api-fallback": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.4.tgz", + "integrity": "sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==", + "license": "MIT", + "dependencies": { + "@types/express-serve-static-core": "*", + "@types/node": "*" + } + }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", + "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==", + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT" + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.6.tgz", + "integrity": "sha512-4fvZhzMeeuBJYZXRXrRIQnvUYfyXwYmLsdiN7XXmVNQKKw1cM8a5WdID0g1hVFZDqT9ZqZEY5pD44p24VS7iZQ==", + "license": "MIT" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "license": "MIT", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "license": "MIT" + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "license": "MIT" + }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-P2dlU/q51fkOc/Gfl3Ul9kicV7l+ra934qBFXCFhrZMOL6du1TM0pm1ThYvENukyOn5h9v+yMJ9Fn5JK4QozrQ==", + "license": "MIT" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "license": "MIT" + }, + "node_modules/@types/d3-random": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.3.tgz", + "integrity": "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.8.tgz", + "integrity": "sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.3.tgz", + "integrity": "sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==", + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "license": "MIT" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.6.tgz", + "integrity": "sha512-5KKk5aKGu2I+O6SONMYSNflgiP0WfZIQvVUMan50wHsLG1G94JlxEVnCpQARfTtzytuY0p/9PXXZb3I7giofIA==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.3.tgz", + "integrity": "sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==", + "license": "MIT" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/eslint": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", + "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", + "license": "MIT", + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", + "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", + "license": "MIT", + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.1.tgz", + "integrity": "sha512-CRICJIl0N5cXDONAdlTv5ShATZ4HEwk6kDDIW2/w9qOWKg+NU/5F8wYRWCrONad0/UKkloNSmmyN/wX4rtpbVA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/express/node_modules/@types/express-serve-static-core": { + "version": "4.19.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", + "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/geojson": { + "version": "7946.0.14", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz", + "integrity": "sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==", + "license": "MIT" + }, + "node_modules/@types/gtag.js": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/@types/gtag.js/-/gtag.js-0.0.12.tgz", + "integrity": "sha512-YQV9bUsemkzG81Ea295/nF/5GijnD2Af7QhEofh7xu+kvCN6RdodgNwwGWXB5GMI3NoyvQo0odNctoH/qLMIpg==", + "license": "MIT" + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/history": { + "version": "4.7.11", + "resolved": "https://registry.npmjs.org/@types/history/-/history-4.7.11.tgz", + "integrity": "sha512-qjDJRrmvBMiTx+jyLxvLfJU7UznFuokDv4f3WRuriHKERccVpFU+8XMQUAbDzoiJCsmexxRExQeMwwCdamSKDA==", + "license": "MIT" + }, + "node_modules/@types/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==", + "license": "MIT" + }, + "node_modules/@types/http-cache-semantics": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", + "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "license": "MIT" + }, + "node_modules/@types/http-proxy": { + "version": "1.17.15", + "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.15.tgz", + "integrity": "sha512-25g5atgiVNTIv0LBDTg1H74Hvayx0ajtJPLLcYE3whFv75J0pWNtOBzaXJQgDTmrX1bx5U9YC2w/n65BN1HwRQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdx": { + "version": "2.0.13", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.13.tgz", + "integrity": "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==", + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.1.tgz", + "integrity": "sha512-p8Yy/8sw1caA8CdRIQBG5tiLHmxtQKObCijiAa9Ez+d4+PRffM4054xbju0msf+cvhJpnFEeNjxmVT/0ipktrg==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.8" + } + }, + "node_modules/@types/node-forge": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz", + "integrity": "sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/prismjs": { + "version": "1.26.5", + "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.5.tgz", + "integrity": "sha512-AUZTa7hQ2KY5L7AmtSiqxlhWxb4ina0yd8hNbl4TWuqnv/pFP0nDMb3YrfSBf4hJVGLh2YEIBfKaBW/9UEl6IQ==", + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.13", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.13.tgz", + "integrity": "sha512-hCZTSvwbzWGvhqxp/RqVqwU999pBf2vp7hzIjiYOsl8wqOmUxkQ6ddw1cV3l8811+kdUFus/q4d1Y3E3SyEifA==", + "license": "MIT" + }, + "node_modules/@types/qs": { + "version": "6.9.17", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.17.tgz", + "integrity": "sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==", + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.12", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", + "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-router": { + "version": "5.1.20", + "resolved": "https://registry.npmjs.org/@types/react-router/-/react-router-5.1.20.tgz", + "integrity": "sha512-jGjmu/ZqS7FjSH6owMcD5qpq19+1RS9DeVRqfl1FeBMxTDQAGwlMWOcs52NDoXaNKyG3d1cYQFMs9rCrb88o9Q==", + "license": "MIT", + "dependencies": { + "@types/history": "^4.7.11", + "@types/react": "*" + } + }, + "node_modules/@types/react-router-config": { + "version": "5.0.11", + "resolved": "https://registry.npmjs.org/@types/react-router-config/-/react-router-config-5.0.11.tgz", + "integrity": "sha512-WmSAg7WgqW7m4x8Mt4N6ZyKz0BubSj/2tVUMsAHp+Yd2AMwcSbeFq9WympT19p5heCFmF97R9eD5uUR/t4HEqw==", + "license": "MIT", + "dependencies": { + "@types/history": "^4.7.11", + "@types/react": "*", + "@types/react-router": "^5.1.0" + } + }, + "node_modules/@types/react-router-dom": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/@types/react-router-dom/-/react-router-dom-5.3.3.tgz", + "integrity": "sha512-kpqnYK4wcdm5UaWI3fLcELopqLrHgLqNsdpHauzlQktfkHL3npOSwtj1Uz9oKBAzs7lFtVkV8j83voAz2D8fhw==", + "license": "MIT", + "dependencies": { + "@types/history": "^4.7.11", + "@types/react": "*", + "@types/react-router": "*" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-index": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.4.tgz", + "integrity": "sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==", + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, + "node_modules/@types/sockjs": { + "version": "0.3.36", + "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz", + "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.5.13", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz", + "integrity": "sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/yargs": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", + "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "license": "ISC" + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "license": "MIT", + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "license": "Apache-2.0", + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "license": "BSD-3-Clause" + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "license": "Apache-2.0" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/address": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/address/-/address-1.2.2.tgz", + "integrity": "sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "license": "MIT", + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/algoliasearch": { + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.24.0.tgz", + "integrity": "sha512-bf0QV/9jVejssFBmz2HQLxUadxk574t4iwjCKp5E7NBzwKkrDEhKPISIIjAU/p6K5qDx3qoeh4+26zWN1jmw3g==", + "license": "MIT", + "dependencies": { + "@algolia/cache-browser-local-storage": "4.24.0", + "@algolia/cache-common": "4.24.0", + "@algolia/cache-in-memory": "4.24.0", + "@algolia/client-account": "4.24.0", + "@algolia/client-analytics": "4.24.0", + "@algolia/client-common": "4.24.0", + "@algolia/client-personalization": "4.24.0", + "@algolia/client-search": "4.24.0", + "@algolia/logger-common": "4.24.0", + "@algolia/logger-console": "4.24.0", + "@algolia/recommend": "4.24.0", + "@algolia/requester-browser-xhr": "4.24.0", + "@algolia/requester-common": "4.24.0", + "@algolia/requester-node-http": "4.24.0", + "@algolia/transporter": "4.24.0" + } + }, + "node_modules/algoliasearch-helper": { + "version": "3.23.0", + "resolved": "https://registry.npmjs.org/algoliasearch-helper/-/algoliasearch-helper-3.23.0.tgz", + "integrity": "sha512-8CK4Gb/ju4OesAYcS+mjBpNiVA7ILWpg7D2vhBZohh0YkG8QT1KZ9LG+8+EntQBUGoKtPy06OFhiwP4f5zzAQg==", + "license": "MIT", + "dependencies": { + "@algolia/events": "^4.0.1" + }, + "peerDependencies": { + "algoliasearch": ">= 3.1 < 6" + } + }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "license": "ISC", + "dependencies": { + "string-width": "^4.1.0" + } + }, + "node_modules/ansi-align/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-html-community": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz", + "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", + "engines": [ + "node >= 0.8.0" + ], + "license": "Apache-2.0", + "bin": { + "ansi-html": "bin/ansi-html" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/asciinema-player": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/asciinema-player/-/asciinema-player-3.10.0.tgz", + "integrity": "sha512-shoOK6F606nDKZxDVM7JuGSCAyWLePoGRFNlV+FqiP5Sqvyn0BlE7wlbjZyd2X4P1iRhv/HKfVNtnQIxmgphRA==", + "license": "Apache-2.0", + "dependencies": { + "@babel/runtime": "^7.21.0", + "solid-js": "^1.3.0" + } + }, + "node_modules/astring": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.9.0.tgz", + "integrity": "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==", + "license": "MIT", + "bin": { + "astring": "bin/astring" + } + }, + "node_modules/autoprefixer": { + "version": "10.4.20", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.20.tgz", + "integrity": "sha512-XY25y5xSv/wEoqzDyXXME4AFfkZI0P23z6Fs3YgymDnKJkCGOnkL0iTxCa85UTqaSgfcqyf3UA6+c7wUvx/16g==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.3", + "caniuse-lite": "^1.0.30001646", + "fraction.js": "^4.3.7", + "normalize-range": "^0.1.2", + "picocolors": "^1.0.1", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/babel-loader": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.2.1.tgz", + "integrity": "sha512-fqe8naHt46e0yIdkjUZYqddSXfej3AHajX+CSO5X7oy0EmPc6o5Xh+RClNoHjnieWz9AW4kZxW9yyFMhVB1QLA==", + "license": "MIT", + "dependencies": { + "find-cache-dir": "^4.0.0", + "schema-utils": "^4.0.0" + }, + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "@babel/core": "^7.12.0", + "webpack": ">=5" + } + }, + "node_modules/babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "license": "MIT", + "dependencies": { + "object.assign": "^4.1.0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2": { + "version": "0.4.12", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.12.tgz", + "integrity": "sha512-CPWT6BwvhrTO2d8QVorhTCQw9Y43zOu7G9HigcfxvepOU6b8o3tcWad6oVgZIsZCTt42FFv97aA7ZJsbM4+8og==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.22.6", + "@babel/helper-define-polyfill-provider": "^0.6.3", + "semver": "^6.3.1" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/babel-plugin-polyfill-corejs3": { + "version": "0.10.6", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.10.6.tgz", + "integrity": "sha512-b37+KR2i/khY5sKmWNVQAnitvquQbNdWy6lJdsr0kmquCKEEUgMKK4SboVM3HtfnZilfjr4MMQ7vY58FVWDtIA==", + "license": "MIT", + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.6.2", + "core-js-compat": "^3.38.0" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/babel-plugin-polyfill-regenerator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.6.3.tgz", + "integrity": "sha512-LiWSbl4CRSIa5x/JAU6jZiG9eit9w6mz+yVMFwDE83LAWvt0AfGBoZ7HS/mkhrKuh2ZlzfVZYKoLjXdqw6Yt7Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.6.3" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/batch": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", + "license": "MIT" + }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/bonjour-service": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", + "integrity": "sha512-3YuAUiSkWykd+2Azjgyxei8OWf8thdn8AITIog2M4UICzoqfjlqr64WIjEXZllf/W6vK1goqleSR6brGomxQqA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "multicast-dns": "^7.2.5" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/boxen": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-6.2.1.tgz", + "integrity": "sha512-H4PEsJXfFI/Pt8sjDWbHlQPx4zL/bvSQjcilJmaulGt5mLDorHOHpmdXAJcBcmru7PhYSp/cDMWRko4ZUMFkSw==", + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.1", + "camelcase": "^6.2.0", + "chalk": "^4.1.2", + "cli-boxes": "^3.0.0", + "string-width": "^5.0.1", + "type-fest": "^2.5.0", + "widest-line": "^4.0.1", + "wrap-ansi": "^8.0.1" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", + "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001669", + "electron-to-chromium": "^1.5.41", + "node-releases": "^2.0.18", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cacheable-lookup": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", + "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + } + }, + "node_modules/cacheable-request": { + "version": "10.2.14", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.14.tgz", + "integrity": "sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==", + "license": "MIT", + "dependencies": { + "@types/http-cache-semantics": "^4.0.2", + "get-stream": "^6.0.1", + "http-cache-semantics": "^4.1.1", + "keyv": "^4.5.3", + "mimic-response": "^4.0.0", + "normalize-url": "^8.0.0", + "responselike": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + } + }, + "node_modules/call-bind": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camel-case": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz", + "integrity": "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==", + "license": "MIT", + "dependencies": { + "pascal-case": "^3.1.2", + "tslib": "^2.0.3" + } + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/caniuse-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-3.0.0.tgz", + "integrity": "sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.0.0", + "caniuse-lite": "^1.0.0", + "lodash.memoize": "^4.1.2", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001680", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001680.tgz", + "integrity": "sha512-rPQy70G6AGUMnbwS1z6Xg+RkHYPAi18ihs47GH0jcxIG7wArmPgY3XbS2sRdBbxJljp3thdT8BIqv9ccCypiPA==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/chevrotain": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-11.0.3.tgz", + "integrity": "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==", + "license": "Apache-2.0", + "dependencies": { + "@chevrotain/cst-dts-gen": "11.0.3", + "@chevrotain/gast": "11.0.3", + "@chevrotain/regexp-to-ast": "11.0.3", + "@chevrotain/types": "11.0.3", + "@chevrotain/utils": "11.0.3", + "lodash-es": "4.17.21" + } + }, + "node_modules/chevrotain-allstar": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/chevrotain-allstar/-/chevrotain-allstar-0.3.1.tgz", + "integrity": "sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==", + "license": "MIT", + "dependencies": { + "lodash-es": "^4.17.21" + }, + "peerDependencies": { + "chevrotain": "^11.0.0" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", + "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==", + "license": "MIT", + "engines": { + "node": ">=6.0" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/clean-css": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-5.3.3.tgz", + "integrity": "sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==", + "license": "MIT", + "dependencies": { + "source-map": "~0.6.0" + }, + "engines": { + "node": ">= 10.0" + } + }, + "node_modules/clean-css/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/cli-boxes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-3.0.0.tgz", + "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-table3": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", + "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==", + "license": "MIT", + "dependencies": { + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "@colors/colors": "1.5.0" + } + }, + "node_modules/cli-table3/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cli-table3/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/collapse-white-space": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.1.0.tgz", + "integrity": "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/colord": { + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", + "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "license": "MIT" + }, + "node_modules/combine-promises": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/combine-promises/-/combine-promises-1.2.0.tgz", + "integrity": "sha512-VcQB1ziGD0NXrhKxiwyNbCDmRzs/OShMs2GqW2DlU2A/Sd0nQxE1oWDAE5O0ygSx5mgQOn9eIFh7yKPgFRVkPQ==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/common-path-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", + "integrity": "sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==", + "license": "ISC" + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compressible/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.5.tgz", + "integrity": "sha512-bQJ0YRck5ak3LgtnpKkiabX5pNF7tMUh1BSy2ZBOTh0Dim0BUu6aPPwByIns6/A5Prh8PufSPerMDUklpzes2Q==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.0.2", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/compression/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/confbox": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", + "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "license": "MIT" + }, + "node_modules/config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "license": "MIT", + "dependencies": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, + "node_modules/configstore": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-6.0.0.tgz", + "integrity": "sha512-cD31W1v3GqUlQvbBCGcXmd2Nj9SvLDOP1oQ0YFuLETufzSPaKp11rYBsSOm7rCsW3OnIRAFM3OxRhceaXNYHkA==", + "license": "BSD-2-Clause", + "dependencies": { + "dot-prop": "^6.0.1", + "graceful-fs": "^4.2.6", + "unique-string": "^3.0.0", + "write-file-atomic": "^3.0.3", + "xdg-basedir": "^5.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/yeoman/configstore?sponsor=1" + } + }, + "node_modules/connect-history-api-fallback": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", + "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/consola": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.2.3.tgz", + "integrity": "sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ==", + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "license": "MIT" + }, + "node_modules/copy-text-to-clipboard": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/copy-text-to-clipboard/-/copy-text-to-clipboard-3.2.0.tgz", + "integrity": "sha512-RnJFp1XR/LOBDckxTib5Qjr/PMfkatD0MUCQgdpqS8MdKiNUzBjAQBEN6oUy+jW7LI93BBG3DtMB2KOOKpGs2Q==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/copy-webpack-plugin": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-11.0.0.tgz", + "integrity": "sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==", + "license": "MIT", + "dependencies": { + "fast-glob": "^3.2.11", + "glob-parent": "^6.0.1", + "globby": "^13.1.1", + "normalize-path": "^3.0.0", + "schema-utils": "^4.0.0", + "serialize-javascript": "^6.0.0" + }, + "engines": { + "node": ">= 14.15.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + } + }, + "node_modules/copy-webpack-plugin/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/copy-webpack-plugin/node_modules/globby": { + "version": "13.2.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", + "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==", + "license": "MIT", + "dependencies": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.3.0", + "ignore": "^5.2.4", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/copy-webpack-plugin/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/core-js": { + "version": "3.39.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.39.0.tgz", + "integrity": "sha512-raM0ew0/jJUqkJ0E6e8UDtl+y/7ktFivgWvqw8dNSQeNWoSDLvQ1H/RN3aPXB9tBd4/FhyR4RDPGhsNIMsAn7g==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-compat": { + "version": "3.39.0", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.39.0.tgz", + "integrity": "sha512-VgEUx3VwlExr5no0tXlBt+silBvhTryPwCXRI2Id1PN8WTKu7MreethvddqOubrYxkFdv/RnYrqlv1sFNAUelw==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.24.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-pure": { + "version": "3.39.0", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.39.0.tgz", + "integrity": "sha512-7fEcWwKI4rJinnK+wLTezeg2smbFFdSBP6E2kQZNbnzM2s1rpKQ6aaRteZSSg7FLU3P0HGGVo/gbpfanU36urg==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "license": "MIT", + "dependencies": { + "layout-base": "^1.0.0" + } + }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "license": "MIT", + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/crypto-random-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-4.0.0.tgz", + "integrity": "sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA==", + "license": "MIT", + "dependencies": { + "type-fest": "^1.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/crypto-random-string/node_modules/type-fest": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz", + "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/css-blank-pseudo": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/css-blank-pseudo/-/css-blank-pseudo-7.0.1.tgz", + "integrity": "sha512-jf+twWGDf6LDoXDUode+nc7ZlrqfaNphrBIBrcmeP3D8yw1uPaix1gCC8LUQUGQ6CycuK2opkbFFWFuq/a94ag==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/css-blank-pseudo/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/css-declaration-sorter": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-7.2.0.tgz", + "integrity": "sha512-h70rUM+3PNFuaBDTLe8wF/cdWu+dOZmb7pJt8Z2sedYbAcQVQV/tEchueg3GWxwqS0cxtbxmaHEdkNACqcvsow==", + "license": "ISC", + "engines": { + "node": "^14 || ^16 || >=18" + }, + "peerDependencies": { + "postcss": "^8.0.9" + } + }, + "node_modules/css-has-pseudo": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/css-has-pseudo/-/css-has-pseudo-7.0.1.tgz", + "integrity": "sha512-EOcoyJt+OsuKfCADgLT7gADZI5jMzIe/AeI6MeAYKiFBDmNmM7kk46DtSfMj5AohUJisqVzopBpnQTlvbyaBWg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/selector-specificity": "^5.0.0", + "postcss-selector-parser": "^7.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/css-has-pseudo/node_modules/@csstools/selector-specificity": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-5.0.0.tgz", + "integrity": "sha512-PCqQV3c4CoVm3kdPhyeZ07VmBRdH2EpMFA/pd9OASpOEC3aXNGoqPDAZ80D0cLpMBxnmk0+yNhGsEx31hq7Gtw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.0.0" + } + }, + "node_modules/css-has-pseudo/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/css-loader": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.11.0.tgz", + "integrity": "sha512-CTJ+AEQJjq5NzLga5pE39qdiSV56F8ywCIsqNIRF0r7BDgWsN25aazToqAFg7ZrtA/U016xudB3ffgweORxX7g==", + "license": "MIT", + "dependencies": { + "icss-utils": "^5.1.0", + "postcss": "^8.4.33", + "postcss-modules-extract-imports": "^3.1.0", + "postcss-modules-local-by-default": "^4.0.5", + "postcss-modules-scope": "^3.2.0", + "postcss-modules-values": "^4.0.0", + "postcss-value-parser": "^4.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/css-minimizer-webpack-plugin": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/css-minimizer-webpack-plugin/-/css-minimizer-webpack-plugin-5.0.1.tgz", + "integrity": "sha512-3caImjKFQkS+ws1TGcFn0V1HyDJFq1Euy589JlD6/3rV2kj+w7r5G9WDMgSHvpvXHNZ2calVypZWuEDQd9wfLg==", + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "cssnano": "^6.0.1", + "jest-worker": "^29.4.3", + "postcss": "^8.4.24", + "schema-utils": "^4.0.1", + "serialize-javascript": "^6.0.1" + }, + "engines": { + "node": ">= 14.15.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "@parcel/css": { + "optional": true + }, + "@swc/css": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "csso": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "lightningcss": { + "optional": true + } + } + }, + "node_modules/css-prefers-color-scheme": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/css-prefers-color-scheme/-/css-prefers-color-scheme-10.0.0.tgz", + "integrity": "sha512-VCtXZAWivRglTZditUfB4StnsWr6YVZ2PRtuxQLKTNRdtAf8tpzaVPE9zXIF3VaSc7O70iK/j1+NXxyQCqdPjQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-tree": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz", + "integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.0.30", + "source-map-js": "^1.0.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssdb": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/cssdb/-/cssdb-8.2.1.tgz", + "integrity": "sha512-KwEPys7lNsC8OjASI8RrmwOYYDcm0JOW9zQhcV83ejYcQkirTEyeAGui8aO2F5PiS6SLpxuTzl6qlMElIdsgIg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + }, + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + } + ], + "license": "MIT-0" + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cssnano": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-6.1.2.tgz", + "integrity": "sha512-rYk5UeX7VAM/u0lNqewCdasdtPK81CgX8wJFLEIXHbV2oldWRgJAsZrdhRXkV1NJzA2g850KiFm9mMU2HxNxMA==", + "license": "MIT", + "dependencies": { + "cssnano-preset-default": "^6.1.2", + "lilconfig": "^3.1.1" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/cssnano" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/cssnano-preset-advanced": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/cssnano-preset-advanced/-/cssnano-preset-advanced-6.1.2.tgz", + "integrity": "sha512-Nhao7eD8ph2DoHolEzQs5CfRpiEP0xa1HBdnFZ82kvqdmbwVBUr2r1QuQ4t1pi+D1ZpqpcO4T+wy/7RxzJ/WPQ==", + "license": "MIT", + "dependencies": { + "autoprefixer": "^10.4.19", + "browserslist": "^4.23.0", + "cssnano-preset-default": "^6.1.2", + "postcss-discard-unused": "^6.0.5", + "postcss-merge-idents": "^6.0.3", + "postcss-reduce-idents": "^6.0.3", + "postcss-zindex": "^6.0.2" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/cssnano-preset-default": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/cssnano-preset-default/-/cssnano-preset-default-6.1.2.tgz", + "integrity": "sha512-1C0C+eNaeN8OcHQa193aRgYexyJtU8XwbdieEjClw+J9d94E41LwT6ivKH0WT+fYwYWB0Zp3I3IZ7tI/BbUbrg==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "css-declaration-sorter": "^7.2.0", + "cssnano-utils": "^4.0.2", + "postcss-calc": "^9.0.1", + "postcss-colormin": "^6.1.0", + "postcss-convert-values": "^6.1.0", + "postcss-discard-comments": "^6.0.2", + "postcss-discard-duplicates": "^6.0.3", + "postcss-discard-empty": "^6.0.3", + "postcss-discard-overridden": "^6.0.2", + "postcss-merge-longhand": "^6.0.5", + "postcss-merge-rules": "^6.1.1", + "postcss-minify-font-values": "^6.1.0", + "postcss-minify-gradients": "^6.0.3", + "postcss-minify-params": "^6.1.0", + "postcss-minify-selectors": "^6.0.4", + "postcss-normalize-charset": "^6.0.2", + "postcss-normalize-display-values": "^6.0.2", + "postcss-normalize-positions": "^6.0.2", + "postcss-normalize-repeat-style": "^6.0.2", + "postcss-normalize-string": "^6.0.2", + "postcss-normalize-timing-functions": "^6.0.2", + "postcss-normalize-unicode": "^6.1.0", + "postcss-normalize-url": "^6.0.2", + "postcss-normalize-whitespace": "^6.0.2", + "postcss-ordered-values": "^6.0.2", + "postcss-reduce-initial": "^6.1.0", + "postcss-reduce-transforms": "^6.0.2", + "postcss-svgo": "^6.0.3", + "postcss-unique-selectors": "^6.0.4" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/cssnano-utils": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cssnano-utils/-/cssnano-utils-4.0.2.tgz", + "integrity": "sha512-ZR1jHg+wZ8o4c3zqf1SIUSTIvm/9mU343FMR6Obe/unskbvpGhZOo1J6d/r8D1pzkRQYuwbcH3hToOuoA2G7oQ==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/csso": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", + "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", + "license": "MIT", + "dependencies": { + "css-tree": "~2.2.0" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/csso/node_modules/css-tree": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", + "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", + "license": "MIT", + "dependencies": { + "mdn-data": "2.0.28", + "source-map-js": "^1.0.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/csso/node_modules/mdn-data": { + "version": "2.0.28", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", + "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==", + "license": "CC0-1.0" + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "license": "MIT" + }, + "node_modules/cytoscape": { + "version": "3.30.3", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.30.3.tgz", + "integrity": "sha512-HncJ9gGJbVtw7YXtIs3+6YAFSSiKsom0amWc33Z7QbylbY2JGMrA0yz4EwrdTScZxnwclXeEZHzO5pxoy0ZE4g==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^2.2.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/cose-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", + "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", + "license": "MIT", + "dependencies": { + "layout-base": "^2.0.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/layout-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", + "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", + "license": "MIT" + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "license": "ISC", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC" + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dagre-d3-es": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.11.tgz", + "integrity": "sha512-tvlJLyQf834SylNKax8Wkzco/1ias1OPw8DcUMDE7oUIoSEW25riQVuiu/0OWEFqT0cxHT3Pa9/D82Jr47IONw==", + "license": "MIT", + "dependencies": { + "d3": "^7.9.0", + "lodash-es": "^4.17.21" + } + }, + "node_modules/dayjs": { + "version": "1.11.13", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.13.tgz", + "integrity": "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg==", + "license": "MIT" + }, + "node_modules/debounce": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/debounce/-/debounce-1.2.1.tgz", + "integrity": "sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decompress-response/node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-gateway": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-6.0.3.tgz", + "integrity": "sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==", + "license": "BSD-2-Clause", + "dependencies": { + "execa": "^5.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/defer-to-connect": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", + "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/delaunator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.1.tgz", + "integrity": "sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, + "node_modules/detect-port": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/detect-port/-/detect-port-1.6.1.tgz", + "integrity": "sha512-CmnVc+Hek2egPx1PeTFVta2W78xy2K/9Rkf6cC4T59S50tVnzKj+tnx5mmx5lwvCkujZ4uRrpRSuV+IVs3f90Q==", + "license": "MIT", + "dependencies": { + "address": "^1.0.1", + "debug": "4" + }, + "bin": { + "detect": "bin/detect-port.js", + "detect-port": "bin/detect-port.js" + }, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "license": "MIT", + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dns-packet": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", + "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==", + "license": "MIT", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dom-converter": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", + "integrity": "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==", + "license": "MIT", + "dependencies": { + "utila": "~0.4" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/dompurify": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.6.tgz", + "integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", + "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "license": "MIT", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dot-prop/node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "license": "MIT" + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.63", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.63.tgz", + "integrity": "sha512-ddeXKuY9BHo/mw145axlyWjlJ1UBt4WK3AlvkT7W2AbqfRQoacVoRUCF6wL3uIx/8wT9oLKXzI+rFqHHscByaA==", + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/emojilib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/emojilib/-/emojilib-2.4.0.tgz", + "integrity": "sha512-5U0rVMU5Y2n2+ykNLQqMoqklN9ICBT/KsvC1Gz6vqHbz2AXXGkG+Pm5rMWk/8Vjrr/mY9985Hi8DYzn1F09Nyw==", + "license": "MIT" + }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/emoticon": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/emoticon/-/emoticon-4.1.0.tgz", + "integrity": "sha512-VWZfnxqwNcc51hIy/sbOdEem6D+cVtpPzEEtVAFdaas30+1dgkyaOQ4sQ6Bp0tOMqWO1v+HQfYaoodOkdhK6SQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.17.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz", + "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz", + "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==", + "license": "MIT" + }, + "node_modules/esast-util-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz", + "integrity": "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/esast-util-from-js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esast-util-from-js/-/esast-util-from-js-2.0.1.tgz", + "integrity": "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "acorn": "^8.0.0", + "esast-util-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-goat": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-4.0.0.tgz", + "integrity": "sha512-2Sd4ShcWxbx6OY1IHyla/CVNwvg7XwZVoXZHcSu9w9SReNP1EzzD5T8NWKIR38fIqEns9kDWKUQTXXAmlDrdPg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-util-attach-comments": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-3.0.0.tgz", + "integrity": "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-build-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-3.0.1.tgz", + "integrity": "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "estree-walker": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-scope": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/estree-util-scope/-/estree-util-scope-1.0.0.tgz", + "integrity": "sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-to-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-2.0.0.tgz", + "integrity": "sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "astring": "^1.8.0", + "source-map": "^0.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-value-to-estree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/estree-util-value-to-estree/-/estree-util-value-to-estree-3.2.1.tgz", + "integrity": "sha512-Vt2UOjyPbNQQgT5eJh+K5aATti0OjCIAGc9SgMdOFYbohuifsWclR74l0iZTJwePMgWYdX1hlVS+dedH9XV8kw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/remcohaszing" + } + }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eta": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/eta/-/eta-2.2.0.tgz", + "integrity": "sha512-UVQ72Rqjy/ZKQalzV5dCCJP80GrmPrMxh6NlNf+erV6ObL0ZFkhCstWRawS85z3smdr3d2wXPsZEY7rDPfGd2g==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + }, + "funding": { + "url": "https://github.com/eta-dev/eta?sponsor=1" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eval": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eval/-/eval-0.1.8.tgz", + "integrity": "sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==", + "dependencies": { + "@types/node": "*", + "require-like": ">= 0.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/express": { + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.10", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/express/node_modules/path-to-regexp": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", + "license": "MIT" + }, + "node_modules/express/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.3.tgz", + "integrity": "sha512-aLrHthzCjH5He4Z2H9YZ+v6Ujb9ocRuW6ZzkJQOrTxleEijANq4v1TsaPaVG1PZcuurEzrLcWRyYBYXD5cEiaw==", + "license": "BSD-3-Clause" + }, + "node_modules/fastq": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fault": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fault/-/fault-2.0.1.tgz", + "integrity": "sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==", + "license": "MIT", + "dependencies": { + "format": "^0.2.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "license": "Apache-2.0", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/feed": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/feed/-/feed-4.2.2.tgz", + "integrity": "sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==", + "license": "MIT", + "dependencies": { + "xml-js": "^1.6.11" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/figures/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/file-loader": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-6.2.0.tgz", + "integrity": "sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw==", + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/file-loader/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/file-loader/node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/file-loader/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, + "node_modules/file-loader/node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/find-cache-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-4.0.0.tgz", + "integrity": "sha512-9ZonPT4ZAK4a+1pUPVPZJapbi7O5qbbJPdYw/NOQWZZbVLdDTYM3A4R9z/DpAM08IDaFGsvPgiGZ82WEwUDWjg==", + "license": "MIT", + "dependencies": { + "common-path-prefix": "^3.0.0", + "pkg-dir": "^7.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "license": "MIT", + "dependencies": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "license": "BSD-3-Clause", + "bin": { + "flat": "cli.js" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data-encoder": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", + "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==", + "license": "MIT", + "engines": { + "node": ">= 14.17" + } + }, + "node_modules/format": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", + "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fraction.js": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", + "integrity": "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==", + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fs-monkey": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.6.tgz", + "integrity": "sha512-b1FMfwetIKymC0eioW7mTywihSQE4oLzQn1dB6rZB5fx/3NpNEdAWeCSMB+60/AeT0TCXsxzAlcYVEFCTAksWg==", + "license": "Unlicense" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-own-enumerable-property-symbols": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz", + "integrity": "sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==", + "license": "ISC" + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/github-slugger": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.5.0.tgz", + "integrity": "sha512-wIh+gKBI9Nshz2o46B0B3f5k/W+WI9ZAv6y5Dn5WJ5SK1t0TnDimB4WE5rmTD05ZAIn8HALCZVmCsvj0w0v0lw==", + "license": "ISC" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "license": "BSD-2-Clause" + }, + "node_modules/global-dirs": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-3.0.1.tgz", + "integrity": "sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==", + "license": "MIT", + "dependencies": { + "ini": "2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/global-dirs/node_modules/ini": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", + "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "license": "MIT", + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/got": { + "version": "12.6.1", + "resolved": "https://registry.npmjs.org/got/-/got-12.6.1.tgz", + "integrity": "sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^5.2.0", + "@szmarczak/http-timer": "^5.0.1", + "cacheable-lookup": "^7.0.0", + "cacheable-request": "^10.2.8", + "decompress-response": "^6.0.0", + "form-data-encoder": "^2.1.2", + "get-stream": "^6.0.1", + "http2-wrapper": "^2.1.10", + "lowercase-keys": "^3.0.0", + "p-cancelable": "^3.0.0", + "responselike": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/got/node_modules/@sindresorhus/is": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.6.0.tgz", + "integrity": "sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/gzip-size": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", + "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "license": "MIT", + "dependencies": { + "duplexer": "^0.1.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hachure-fill": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", + "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", + "license": "MIT" + }, + "node_modules/handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-yarn": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-3.0.0.tgz", + "integrity": "sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hast-util-from-parse5": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.2.tgz", + "integrity": "sha512-SfMzfdAi/zAoZ1KkFEyyeXBn7u/ShQrfd675ZEE9M3qj+PMFX05xubzRyF76CCSJu8au9jgVxDV1+okFvgZU4A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "hastscript": "^9.0.0", + "property-information": "^6.0.0", + "vfile": "^6.0.0", + "vfile-location": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", + "integrity": "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-from-parse5": "^8.0.0", + "hast-util-to-parse5": "^8.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "parse5": "^7.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-estree": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.0.tgz", + "integrity": "sha512-lfX5g6hqVh9kjS/B9E2gSkvHH4SZNiQFiqWS0x9fENzEl+8W12RqdRxX6d/Cwxi30tPQs3bIO+aolQJNp1bIyw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-attach-comments": "^3.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.4.0", + "unist-util-position": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-estree/node_modules/inline-style-parser": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", + "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==", + "license": "MIT" + }, + "node_modules/hast-util-to-estree/node_modules/style-to-object": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.4.4.tgz", + "integrity": "sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.1.1" + } + }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.2.tgz", + "integrity": "sha512-1ngXYb+V9UT5h+PxNRa1O1FYguZK/XL+gkeqvp7EdHlB9oHUG0eYRo/vY5inBdcqo3RkPMC58/H94HvkbfGdyg==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-parse5": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.0.tgz", + "integrity": "sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.0.tgz", + "integrity": "sha512-jzaLBGavEDKHrc5EfFImKN7nZKKBdSLIdGvCwDZ9TfzbF2ffXiov8CKE445L2Z1Ek2t/m4SKQ2j6Ipv7NyUolw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/history": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz", + "integrity": "sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.1.2", + "loose-envify": "^1.2.0", + "resolve-pathname": "^3.0.0", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0", + "value-equal": "^1.0.1" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "license": "BSD-3-Clause", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "node_modules/hpack.js/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/hpack.js/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/hpack.js/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/hpack.js/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/htm": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz", + "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==", + "license": "Apache-2.0" + }, + "node_modules/html-entities": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", + "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "license": "MIT" + }, + "node_modules/html-minifier-terser": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-7.2.0.tgz", + "integrity": "sha512-tXgn3QfqPIpGl9o+K5tpcj3/MN4SfLtsx2GWwBC3SSd0tXQGyF3gsSqad8loJgKZGM3ZxbYDd5yhiBIdWpmvLA==", + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "~5.3.2", + "commander": "^10.0.0", + "entities": "^4.4.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.15.1" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, + "node_modules/html-minifier-terser/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/html-tags": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-3.3.1.tgz", + "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/html-webpack-plugin": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-5.6.3.tgz", + "integrity": "sha512-QSf1yjtSAsmf7rYBV7XX86uua4W/vkhIt0xNXKbsi2foEeW7vjJQz4bhnpL3xH+l1ryl1680uNv968Z+X6jSYg==", + "license": "MIT", + "dependencies": { + "@types/html-minifier-terser": "^6.0.0", + "html-minifier-terser": "^6.0.2", + "lodash": "^4.17.21", + "pretty-error": "^4.0.0", + "tapable": "^2.0.0" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/html-webpack-plugin" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.20.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/html-webpack-plugin/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/html-webpack-plugin/node_modules/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==", + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "^5.2.2", + "commander": "^8.3.0", + "he": "^1.2.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.10.0" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", + "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", + "license": "BSD-2-Clause" + }, + "node_modules/http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-parser-js": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz", + "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==", + "license": "MIT" + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "license": "MIT", + "dependencies": { + "@types/http-proxy": "^1.17.8", + "http-proxy": "^1.18.1", + "is-glob": "^4.0.1", + "is-plain-obj": "^3.0.0", + "micromatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "@types/express": "^4.17.13" + }, + "peerDependenciesMeta": { + "@types/express": { + "optional": true + } + } + }, + "node_modules/http-proxy-middleware/node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/http2-wrapper": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.1.tgz", + "integrity": "sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==", + "license": "MIT", + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.2.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/icss-utils": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", + "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==", + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/image-size": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-2.0.2.tgz", + "integrity": "sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w==", + "license": "MIT", + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=16.x" + } + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-lazy": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-4.0.0.tgz", + "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/infima": { + "version": "0.2.0-alpha.45", + "resolved": "https://registry.npmjs.org/infima/-/infima-0.2.0-alpha.45.tgz", + "integrity": "sha512-uyH0zfr1erU1OohLk0fT4Rrb94AOhguWNOcD9uGrSpRvNB+6gZXUoJX5J0NtvzBO10YZ9PgvA4NFgt+fYg8ojw==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/inline-style-parser": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.4.tgz", + "integrity": "sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==", + "license": "MIT" + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.0.0" + } + }, + "node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-ci": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz", + "integrity": "sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==", + "license": "MIT", + "dependencies": { + "ci-info": "^3.2.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, + "node_modules/is-core-module": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", + "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-installed-globally": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.4.0.tgz", + "integrity": "sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==", + "license": "MIT", + "dependencies": { + "global-dirs": "^3.0.0", + "is-path-inside": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-npm": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-6.0.0.tgz", + "integrity": "sha512-JEjxbSmtPSt1c8XTkVrlujcXdKV1/tvuQ7GwKcAlyiVLeYFQ2VHat8xfrDJsIkhCdF/tZ7CiIR3sy141c6+gPQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-regexp": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz", + "integrity": "sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", + "license": "MIT" + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-yarn-global": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.4.1.tgz", + "integrity": "sha512-/kppl+R+LO5VmhYSEWARUFjodS25D68gvj8W7z0I7OWhUla5xWu8KL6CtB2V0R6yqhnRgbcaREMr4EEM6htLPQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/jiti": { + "version": "1.21.6", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz", + "integrity": "sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==", + "license": "MIT", + "bin": { + "jiti": "bin/jiti.js" + } + }, + "node_modules/joi": { + "version": "17.13.3", + "resolved": "https://registry.npmjs.org/joi/-/joi-17.13.3.tgz", + "integrity": "sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^9.3.0", + "@hapi/topo": "^5.1.0", + "@sideway/address": "^4.1.5", + "@sideway/formula": "^3.0.1", + "@sideway/pinpoint": "^2.0.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/katex": { + "version": "0.16.11", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.11.tgz", + "integrity": "sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/katex/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/kolorist": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/kolorist/-/kolorist-1.8.0.tgz", + "integrity": "sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==", + "license": "MIT" + }, + "node_modules/langium": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/langium/-/langium-3.3.1.tgz", + "integrity": "sha512-QJv/h939gDpvT+9SiLVlY7tZC3xB2qK57v0J04Sh9wpMb6MP1q8gB21L3WIo8T5P1MSMg3Ep14L7KkDCFG3y4w==", + "license": "MIT", + "dependencies": { + "chevrotain": "~11.0.3", + "chevrotain-allstar": "~0.3.0", + "vscode-languageserver": "~9.0.1", + "vscode-languageserver-textdocument": "~1.0.11", + "vscode-uri": "~3.0.8" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/latest-version": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-7.0.0.tgz", + "integrity": "sha512-KvNT4XqAMzdcL6ka6Tl3i2lYeFDgXNCuIX+xNx6ZMVR1dFq+idXd9FLKNMOIx0t9mJ9/HudyX4oZWXZQ0UJHeg==", + "license": "MIT", + "dependencies": { + "package-json": "^8.1.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/launch-editor": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.9.1.tgz", + "integrity": "sha512-Gcnl4Bd+hRO9P9icCP/RVVT2o8SFlPXofuCxvA2SaZuH45whSvf5p8x5oih5ftLiVhEI4sp5xDY+R+b3zJBh5w==", + "license": "MIT", + "dependencies": { + "picocolors": "^1.0.0", + "shell-quote": "^1.8.1" + } + }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", + "license": "MIT" + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lilconfig": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.2.tgz", + "integrity": "sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/loader-runner": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "license": "MIT", + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, + "node_modules/local-pkg": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.1.tgz", + "integrity": "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ==", + "license": "MIT", + "dependencies": { + "mlly": "^1.7.3", + "pkg-types": "^1.2.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "license": "MIT", + "dependencies": { + "p-locate": "^6.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", + "license": "MIT" + }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "license": "MIT" + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lower-case": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", + "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.3" + } + }, + "node_modules/lowercase-keys": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", + "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lunr-languages": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/lunr-languages/-/lunr-languages-1.14.0.tgz", + "integrity": "sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==", + "license": "MPL-1.1" + }, + "node_modules/mark.js": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz", + "integrity": "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==", + "license": "MIT" + }, + "node_modules/markdown-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-2.0.0.tgz", + "integrity": "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/marked": { + "version": "15.0.12", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.12.tgz", + "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/mdast-util-directive": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-directive/-/mdast-util-directive-3.0.0.tgz", + "integrity": "sha512-JUpYOqKI4mM3sZcNxmF/ox04XYFFkNwr0CFlrQIkCwbvH0xzMCqkMqAde9wRd80VAhaUrwFwKm2nxretdT1h7Q==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.1.tgz", + "integrity": "sha512-SG21kZHGC3XRTSUhtofZkBzZTJNM5ecCi0SK2IMKmSXR8vO3peL+kb1O0z7Zl83jKtutG4k5Wv/W7V3/YHvzPA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.2.tgz", + "integrity": "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-from-markdown/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/mdast-util-frontmatter": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-frontmatter/-/mdast-util-frontmatter-2.0.1.tgz", + "integrity": "sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "escape-string-regexp": "^5.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-extension-frontmatter": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-frontmatter/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.0.0.tgz", + "integrity": "sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/mdast-util-gfm-autolink-literal/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.0.0.tgz", + "integrity": "sha512-5jOT2boTSVkMnQ7LTrd6n/18kqwjmuYqo7JUPe+tRCY6O7dAuTFMtTPauYYrMPpox9hlN0uOx/FL8XvEfG9/mQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.0.tgz", + "integrity": "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", + "integrity": "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.1.3.tgz", + "integrity": "sha512-bfOjvNt+1AcbPLTFMFWY149nJz0OjmewJs3LQQ5pIyVGxP4CdOqNVJL6kTaM5c68p8q82Xv3nCyFfUnuEcH3UQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", + "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdn-data": { + "version": "2.0.30", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz", + "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", + "license": "CC0-1.0" + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/memfs": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-3.5.3.tgz", + "integrity": "sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==", + "license": "Unlicense", + "dependencies": { + "fs-monkey": "^1.0.4" + }, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/mermaid": { + "version": "11.6.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.6.0.tgz", + "integrity": "sha512-PE8hGUy1LDlWIHWBP05SFdqUHGmRcCcK4IzpOKPE35eOw+G9zZgcnMpyunJVUEOgb//KBORPjysKndw8bFLuRg==", + "license": "MIT", + "dependencies": { + "@braintree/sanitize-url": "^7.0.4", + "@iconify/utils": "^2.1.33", + "@mermaid-js/parser": "^0.4.0", + "@types/d3": "^7.4.3", + "cytoscape": "^3.29.3", + "cytoscape-cose-bilkent": "^4.1.0", + "cytoscape-fcose": "^2.2.0", + "d3": "^7.9.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.11", + "dayjs": "^1.11.13", + "dompurify": "^3.2.4", + "katex": "^0.16.9", + "khroma": "^2.1.0", + "lodash-es": "^4.17.21", + "marked": "^15.0.7", + "roughjs": "^4.6.6", + "stylis": "^4.3.6", + "ts-dedent": "^2.2.0", + "uuid": "^11.1.0" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromark": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.1.tgz", + "integrity": "sha512-eBPdkcoCNvYcxQOAKAlceo5SNdzZWfF+FcSupREAzdAh9rRmE239CEQAiTwIgblwnoM8zzj35sZ5ZwvSEOF6Kw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.2.tgz", + "integrity": "sha512-FKjQKbxd1cibWMM1P9N+H8TwlgGgSkWZMmfuVucLCHaYqeSvJ0hFeHsIa65pA2nYbes0f8LDHPMrd9X7Ujxg9w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-directive": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-directive/-/micromark-extension-directive-3.0.2.tgz", + "integrity": "sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-directive/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-frontmatter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-frontmatter/-/micromark-extension-frontmatter-2.0.0.tgz", + "integrity": "sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==", + "license": "MIT", + "dependencies": { + "fault": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-frontmatter/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-frontmatter/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-footnote/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-footnote/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.0.tgz", + "integrity": "sha512-Ub2ncQv+fwD70/l4ou27b4YzfNaCJOvyX4HxXU15m7mpYY+rjuWzsLIPZHJL253Z643RpbcP1oeIJlQ/SKW67g==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-table/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-table/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-task-list-item/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm-task-list-item/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-mdx-expression": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.0.tgz", + "integrity": "sha512-sI0nwhUDz97xyzqJAbHQhp5TfaxEvZZZ2JDqUo+7NvyIYG6BZ5CPPqj2ogUoPJlmXHBnyZUzISg9+oUmU6tUjQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-expression/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-expression/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-expression/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.1.tgz", + "integrity": "sha512-vNuFb9czP8QCtAQcEJn0UJQJZA8Dk6DXKBqx+bg/w0WGuSxDxNr7hErW89tHUY31dUW4NqEOWwmEUNhjTFmHkg==", + "license": "MIT", + "dependencies": { + "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-jsx/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-jsx/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-jsx/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-extension-mdx-md": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", + "integrity": "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", + "integrity": "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==", + "license": "MIT", + "dependencies": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.0", + "micromark-extension-mdx-md": "^2.0.0", + "micromark-extension-mdxjs-esm": "^3.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", + "integrity": "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdxjs-esm/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.2.tgz", + "integrity": "sha512-5E5I2pFzJyg2CtemqAbcyCktpHXuJbABnsb32wX2U8IQKhhVFBqkcZR5LRm1WVoFqa4kTueZK4abep7wdo9nrw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-factory-space": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-1.1.0.tgz", + "integrity": "sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-factory-space/node_modules/micromark-util-types": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-1.1.0.tgz", + "integrity": "sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-character": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-1.2.0.tgz", + "integrity": "sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-character/node_modules/micromark-util-types": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-1.1.0.tgz", + "integrity": "sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.2.tgz", + "integrity": "sha512-Fk+xmBrOv9QZnEDguL9OI9/NQQp6Hz4FuQ4YmCb/5V7+9eAh1s6AYSvL20kHkD67YIg7EpE54TiSlcsf3vyZgA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-util-events-to-acorn/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-normalize-identifier/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.3.tgz", + "integrity": "sha512-VXJJuNxYWSoYL6AJ6OQECCFGhIU2GGHMw8tahogePBrjkG8aCCas3ibkp7RnVOSTClg2is05/R7maAhF1XyQMg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-symbol": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-1.1.0.tgz", + "integrity": "sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.1.tgz", + "integrity": "sha512-534m2WhVTddrcKVepwmVEVnUAmtrx9bfIjNoQHRqfnvdaHQiFytEhJoTgpWJvDEXCO5gLTQh3wYC1PgOJA4NSQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark/node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark/node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark/node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "license": "MIT", + "dependencies": { + "mime-db": "~1.33.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", + "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mini-css-extract-plugin": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.9.2.tgz", + "integrity": "sha512-GJuACcS//jtq4kCtd5ii/M0SZf7OZRH+BxdqXZHaJfb8TJiVl+NgQRPwiYt2EuqeSkNydn/7vP+bcE27C5mb9w==", + "license": "MIT", + "dependencies": { + "schema-utils": "^4.0.0", + "tapable": "^2.2.1" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "license": "ISC" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mlly": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.7.3.tgz", + "integrity": "sha512-xUsx5n/mN0uQf4V548PKQ+YShA4/IW0KI1dZhrNrPCLG+xizETbHTkOa1f8/xut9JRPp8kQuMnz0oqwkTiLo/A==", + "license": "MIT", + "dependencies": { + "acorn": "^8.14.0", + "pathe": "^1.1.2", + "pkg-types": "^1.2.1", + "ufo": "^1.5.4" + } + }, + "node_modules/mrmime": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", + "integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/multicast-dns": { + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz", + "integrity": "sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==", + "license": "MIT", + "dependencies": { + "dns-packet": "^5.2.2", + "thunky": "^1.0.2" + }, + "bin": { + "multicast-dns": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT" + }, + "node_modules/no-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", + "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", + "license": "MIT", + "dependencies": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + } + }, + "node_modules/node-emoji": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-2.1.3.tgz", + "integrity": "sha512-E2WEOVsgs7O16zsURJ/eH8BqhF029wGpEOnv7Urwdo2wmQanOACwJQh0devF9D9RhoZru0+9JXIS0dBXIAz+lA==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.6.0", + "char-regex": "^1.0.2", + "emojilib": "^2.4.0", + "skin-tone": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-releases": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", + "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-url": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-8.0.1.tgz", + "integrity": "sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nprogress": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz", + "integrity": "sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==", + "license": "MIT" + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/null-loader": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/null-loader/-/null-loader-4.0.1.tgz", + "integrity": "sha512-pxqVbi4U6N26lq+LmgIbB5XATP0VdZKOG25DhHi8btMmJJefGArFyDg1yc4U3hWCJbMqSrw0qyrz1UQX+qYXqg==", + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/null-loader/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/null-loader/node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/null-loader/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, + "node_modules/null-loader/node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", + "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.assign": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", + "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.5", + "define-properties": "^1.2.1", + "has-symbols": "^1.0.3", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "license": "MIT" + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "license": "MIT", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "license": "(WTFPL OR MIT)", + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/p-cancelable": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", + "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==", + "license": "MIT", + "engines": { + "node": ">=12.20" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "license": "MIT", + "dependencies": { + "p-limit": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "license": "MIT", + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/package-json": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-8.1.1.tgz", + "integrity": "sha512-cbH9IAIJHNj9uXi196JVsRlt7cHKak6u/e6AkL/bkRelZ7rlL3X1YKxsZwa36xipOEKAsdtmaG6aAJoM1fx2zA==", + "license": "MIT", + "dependencies": { + "got": "^12.1.0", + "registry-auth-token": "^5.0.1", + "registry-url": "^6.0.0", + "semver": "^7.3.7" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-manager-detector": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-0.2.4.tgz", + "integrity": "sha512-H/OUu9/zUfP89z1APcBf2X8Us0tt8dUK4lUmKqz12QNXif3DxAs1/YqjGtcutZi1zQqeNQRWr9C+EbQnnvSSFA==", + "license": "MIT" + }, + "node_modules/param-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", + "integrity": "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==", + "license": "MIT", + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-entities": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.1.tgz", + "integrity": "sha512-SWzvYcSJh4d/SGLIOQfZ/CoNv6BTlI6YEQ7Nj82oDVnRpwe/Z/F1EMx42x3JAOwGBlCjeCH0BRJQbQ/opHL17w==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse-numeric-range": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz", + "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ==", + "license": "ISC" + }, + "node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "license": "MIT", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/pascal-case": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", + "integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==", + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/path-data-parser": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", + "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", + "license": "MIT" + }, + "node_modules/path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "license": "(WTFPL OR MIT)" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz", + "integrity": "sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==", + "license": "MIT", + "dependencies": { + "isarray": "0.0.1" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pathe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", + "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkg-dir": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-7.0.0.tgz", + "integrity": "sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA==", + "license": "MIT", + "dependencies": { + "find-up": "^6.3.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-types": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.2.1.tgz", + "integrity": "sha512-sQoqa8alT3nHjGuTjuKgOnvjo4cljkufdtLMnO2LBP/wRwuDlo1tkaEdMxCRhyGRPacv/ztlZgDPm2b7FAmEvw==", + "license": "MIT", + "dependencies": { + "confbox": "^0.1.8", + "mlly": "^1.7.2", + "pathe": "^1.1.2" + } + }, + "node_modules/points-on-curve": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", + "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", + "license": "MIT" + }, + "node_modules/points-on-path": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", + "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", + "license": "MIT", + "dependencies": { + "path-data-parser": "0.1.0", + "points-on-curve": "0.2.0" + } + }, + "node_modules/postcss": { + "version": "8.4.49", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", + "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-attribute-case-insensitive": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-7.0.1.tgz", + "integrity": "sha512-Uai+SupNSqzlschRyNx3kbCTWgY/2hcwtHEI/ej2LJWc9JJ77qKgGptd8DHwY1mXtZ7Aoh4z4yxfwMBue9eNgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-attribute-case-insensitive/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-calc": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-9.0.1.tgz", + "integrity": "sha512-TipgjGyzP5QzEhsOZUaIkeO5mKeMFpebWzRogWG/ysonUlnHcq5aJe0jOjpfzUU8PeSaBQnrE8ehR0QA5vs8PQ==", + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.0.11", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.2.2" + } + }, + "node_modules/postcss-clamp": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/postcss-clamp/-/postcss-clamp-4.1.0.tgz", + "integrity": "sha512-ry4b1Llo/9zz+PKC+030KUnPITTJAHeOwjfAyyB60eT0AorGLdzp52s31OsPRHRf8NchkgFoG2y6fCfn1IV1Ow==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=7.6.0" + }, + "peerDependencies": { + "postcss": "^8.4.6" + } + }, + "node_modules/postcss-color-functional-notation": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/postcss-color-functional-notation/-/postcss-color-functional-notation-7.0.6.tgz", + "integrity": "sha512-wLXvm8RmLs14Z2nVpB4CWlnvaWPRcOZFltJSlcbYwSJ1EDZKsKDhPKIMecCnuU054KSmlmubkqczmm6qBPCBhA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-color-hex-alpha": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/postcss-color-hex-alpha/-/postcss-color-hex-alpha-10.0.0.tgz", + "integrity": "sha512-1kervM2cnlgPs2a8Vt/Qbe5cQ++N7rkYo/2rz2BkqJZIHQwaVuJgQH38REHrAi4uM0b1fqxMkWYmese94iMp3w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-color-rebeccapurple": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/postcss-color-rebeccapurple/-/postcss-color-rebeccapurple-10.0.0.tgz", + "integrity": "sha512-JFta737jSP+hdAIEhk1Vs0q0YF5P8fFcj+09pweS8ktuGuZ8pPlykHsk6mPxZ8awDl4TrcxUqJo9l1IhVr/OjQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-colormin": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-6.1.0.tgz", + "integrity": "sha512-x9yX7DOxeMAR+BgGVnNSAxmAj98NX/YxEMNFP+SDCEeNLb2r3i6Hh1ksMsnW8Ub5SLCpbescQqn9YEbE9554Sw==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "caniuse-api": "^3.0.0", + "colord": "^2.9.3", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-convert-values": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-6.1.0.tgz", + "integrity": "sha512-zx8IwP/ts9WvUM6NkVSkiU902QZL1bwPhaVaLynPtCsOTqp+ZKbNi+s6XJg3rfqpKGA/oc7Oxk5t8pOQJcwl/w==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-custom-media": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/postcss-custom-media/-/postcss-custom-media-11.0.5.tgz", + "integrity": "sha512-SQHhayVNgDvSAdX9NQ/ygcDQGEY+aSF4b/96z7QUX6mqL5yl/JgG/DywcF6fW9XbnCRE+aVYk+9/nqGuzOPWeQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/cascade-layer-name-parser": "^2.0.4", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/media-query-list-parser": "^4.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-custom-properties": { + "version": "14.0.4", + "resolved": "https://registry.npmjs.org/postcss-custom-properties/-/postcss-custom-properties-14.0.4.tgz", + "integrity": "sha512-QnW8FCCK6q+4ierwjnmXF9Y9KF8q0JkbgVfvQEMa93x1GT8FvOiUevWCN2YLaOWyByeDX8S6VFbZEeWoAoXs2A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/cascade-layer-name-parser": "^2.0.4", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-custom-selectors": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/postcss-custom-selectors/-/postcss-custom-selectors-8.0.4.tgz", + "integrity": "sha512-ASOXqNvDCE0dAJ/5qixxPeL1aOVGHGW2JwSy7HyjWNbnWTQCl+fDc968HY1jCmZI0+BaYT5CxsOiUhavpG/7eg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/cascade-layer-name-parser": "^2.0.4", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-custom-selectors/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-dir-pseudo-class": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/postcss-dir-pseudo-class/-/postcss-dir-pseudo-class-9.0.1.tgz", + "integrity": "sha512-tRBEK0MHYvcMUrAuYMEOa0zg9APqirBcgzi6P21OhxtJyJADo/SWBwY1CAwEohQ/6HDaa9jCjLRG7K3PVQYHEA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-dir-pseudo-class/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-discard-comments": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-6.0.2.tgz", + "integrity": "sha512-65w/uIqhSBBfQmYnG92FO1mWZjJ4GL5b8atm5Yw2UgrwD7HiNiSSNwJor1eCFGzUgYnN/iIknhNRVqjrrpuglw==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-discard-duplicates": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-6.0.3.tgz", + "integrity": "sha512-+JA0DCvc5XvFAxwx6f/e68gQu/7Z9ud584VLmcgto28eB8FqSFZwtrLwB5Kcp70eIoWP/HXqz4wpo8rD8gpsTw==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-discard-empty": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-6.0.3.tgz", + "integrity": "sha512-znyno9cHKQsK6PtxL5D19Fj9uwSzC2mB74cpT66fhgOadEUPyXFkbgwm5tvc3bt3NAy8ltE5MrghxovZRVnOjQ==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-discard-overridden": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-6.0.2.tgz", + "integrity": "sha512-j87xzI4LUggC5zND7KdjsI25APtyMuynXZSujByMaav2roV6OZX+8AaCUcZSWqckZpjAjRyFDdpqybgjFO0HJQ==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-discard-unused": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/postcss-discard-unused/-/postcss-discard-unused-6.0.5.tgz", + "integrity": "sha512-wHalBlRHkaNnNwfC8z+ppX57VhvS+HWgjW508esjdaEYr3Mx7Gnn2xA4R/CKf5+Z9S5qsqC+Uzh4ueENWwCVUA==", + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.0.16" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-double-position-gradients": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/postcss-double-position-gradients/-/postcss-double-position-gradients-6.0.0.tgz", + "integrity": "sha512-JkIGah3RVbdSEIrcobqj4Gzq0h53GG4uqDPsho88SgY84WnpkTpI0k50MFK/sX7XqVisZ6OqUfFnoUO6m1WWdg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-focus-visible": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/postcss-focus-visible/-/postcss-focus-visible-10.0.1.tgz", + "integrity": "sha512-U58wyjS/I1GZgjRok33aE8juW9qQgQUNwTSdxQGuShHzwuYdcklnvK/+qOWX1Q9kr7ysbraQ6ht6r+udansalA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-focus-visible/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-focus-within": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/postcss-focus-within/-/postcss-focus-within-9.0.1.tgz", + "integrity": "sha512-fzNUyS1yOYa7mOjpci/bR+u+ESvdar6hk8XNK/TRR0fiGTp2QT5N+ducP0n3rfH/m9I7H/EQU6lsa2BrgxkEjw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-focus-within/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-font-variant": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-font-variant/-/postcss-font-variant-5.0.0.tgz", + "integrity": "sha512-1fmkBaCALD72CK2a9i468mA/+tr9/1cBxRRMXOUaZqO43oWPR5imcyPjXwuv7PXbCid4ndlP5zWhidQVVa3hmA==", + "license": "MIT", + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-gap-properties": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/postcss-gap-properties/-/postcss-gap-properties-6.0.0.tgz", + "integrity": "sha512-Om0WPjEwiM9Ru+VhfEDPZJAKWUd0mV1HmNXqp2C29z80aQ2uP9UVhLc7e3aYMIor/S5cVhoPgYQ7RtfeZpYTRw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-image-set-function": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-image-set-function/-/postcss-image-set-function-7.0.0.tgz", + "integrity": "sha512-QL7W7QNlZuzOwBTeXEmbVckNt1FSmhQtbMRvGGqqU4Nf4xk6KUEQhAoWuMzwbSv5jxiRiSZ5Tv7eiDB9U87znA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/utilities": "^2.0.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-lab-function": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/postcss-lab-function/-/postcss-lab-function-7.0.6.tgz", + "integrity": "sha512-HPwvsoK7C949vBZ+eMyvH2cQeMr3UREoHvbtra76/UhDuiViZH6pir+z71UaJQohd7VDSVUdR6TkWYKExEc9aQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/css-color-parser": "^3.0.6", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/utilities": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-loader": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-7.3.4.tgz", + "integrity": "sha512-iW5WTTBSC5BfsBJ9daFMPVrLT36MrNiC6fqOZTTaHjBNX6Pfd5p+hSBqe/fEeNd7pc13QiAyGt7VdGMw4eRC4A==", + "license": "MIT", + "dependencies": { + "cosmiconfig": "^8.3.5", + "jiti": "^1.20.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">= 14.15.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "postcss": "^7.0.0 || ^8.0.1", + "webpack": "^5.0.0" + } + }, + "node_modules/postcss-logical": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/postcss-logical/-/postcss-logical-8.0.0.tgz", + "integrity": "sha512-HpIdsdieClTjXLOyYdUPAX/XQASNIwdKt5hoZW08ZOAiI+tbV0ta1oclkpVkW5ANU+xJvk3KkA0FejkjGLXUkg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-merge-idents": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/postcss-merge-idents/-/postcss-merge-idents-6.0.3.tgz", + "integrity": "sha512-1oIoAsODUs6IHQZkLQGO15uGEbK3EAl5wi9SS8hs45VgsxQfMnxvt+L+zIr7ifZFIH14cfAeVe2uCTa+SPRa3g==", + "license": "MIT", + "dependencies": { + "cssnano-utils": "^4.0.2", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-merge-longhand": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-6.0.5.tgz", + "integrity": "sha512-5LOiordeTfi64QhICp07nzzuTDjNSO8g5Ksdibt44d+uvIIAE1oZdRn8y/W5ZtYgRH/lnLDlvi9F8btZcVzu3w==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0", + "stylehacks": "^6.1.1" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-merge-rules": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-6.1.1.tgz", + "integrity": "sha512-KOdWF0gju31AQPZiD+2Ar9Qjowz1LTChSjFFbS+e2sFgc4uHOp3ZvVX4sNeTlk0w2O31ecFGgrFzhO0RSWbWwQ==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "caniuse-api": "^3.0.0", + "cssnano-utils": "^4.0.2", + "postcss-selector-parser": "^6.0.16" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-minify-font-values": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-6.1.0.tgz", + "integrity": "sha512-gklfI/n+9rTh8nYaSJXlCo3nOKqMNkxuGpTn/Qm0gstL3ywTr9/WRKznE+oy6fvfolH6dF+QM4nCo8yPLdvGJg==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-minify-gradients": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-6.0.3.tgz", + "integrity": "sha512-4KXAHrYlzF0Rr7uc4VrfwDJ2ajrtNEpNEuLxFgwkhFZ56/7gaE4Nr49nLsQDZyUe+ds+kEhf+YAUolJiYXF8+Q==", + "license": "MIT", + "dependencies": { + "colord": "^2.9.3", + "cssnano-utils": "^4.0.2", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-minify-params": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-6.1.0.tgz", + "integrity": "sha512-bmSKnDtyyE8ujHQK0RQJDIKhQ20Jq1LYiez54WiaOoBtcSuflfK3Nm596LvbtlFcpipMjgClQGyGr7GAs+H1uA==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "cssnano-utils": "^4.0.2", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-minify-selectors": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-6.0.4.tgz", + "integrity": "sha512-L8dZSwNLgK7pjTto9PzWRoMbnLq5vsZSTu8+j1P/2GB8qdtGQfn+K1uSvFgYvgh83cbyxT5m43ZZhUMTJDSClQ==", + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.0.16" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-modules-extract-imports": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.1.0.tgz", + "integrity": "sha512-k3kNe0aNFQDAZGbin48pL2VNidTF0w4/eASDsxlyspobzU3wZQLOGj7L9gfRe0Jo9/4uud09DsjFNH7winGv8Q==", + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-local-by-default": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.1.0.tgz", + "integrity": "sha512-rm0bdSv4jC3BDma3s9H19ZddW0aHX6EoqwDYU2IfZhRN+53QrufTRo2IdkAbRqLx4R2IYbZnbjKKxg4VN5oU9Q==", + "license": "MIT", + "dependencies": { + "icss-utils": "^5.0.0", + "postcss-selector-parser": "^7.0.0", + "postcss-value-parser": "^4.1.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-local-by-default/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-modules-scope": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-3.2.1.tgz", + "integrity": "sha512-m9jZstCVaqGjTAuny8MdgE88scJnCiQSlSrOWcTQgM2t32UBe+MUmFSO5t7VMSfAf/FJKImAxBav8ooCHJXCJA==", + "license": "ISC", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-scope/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-modules-values": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz", + "integrity": "sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==", + "license": "ISC", + "dependencies": { + "icss-utils": "^5.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-nesting": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/postcss-nesting/-/postcss-nesting-13.0.1.tgz", + "integrity": "sha512-VbqqHkOBOt4Uu3G8Dm8n6lU5+9cJFxiuty9+4rcoyRPO9zZS1JIs6td49VIoix3qYqELHlJIn46Oih9SAKo+yQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/selector-resolve-nested": "^3.0.0", + "@csstools/selector-specificity": "^5.0.0", + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-nesting/node_modules/@csstools/selector-resolve-nested": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@csstools/selector-resolve-nested/-/selector-resolve-nested-3.0.0.tgz", + "integrity": "sha512-ZoK24Yku6VJU1gS79a5PFmC8yn3wIapiKmPgun0hZgEI5AOqgH2kiPRsPz1qkGv4HL+wuDLH83yQyk6inMYrJQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.0.0" + } + }, + "node_modules/postcss-nesting/node_modules/@csstools/selector-specificity": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-5.0.0.tgz", + "integrity": "sha512-PCqQV3c4CoVm3kdPhyeZ07VmBRdH2EpMFA/pd9OASpOEC3aXNGoqPDAZ80D0cLpMBxnmk0+yNhGsEx31hq7Gtw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.0.0" + } + }, + "node_modules/postcss-nesting/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-normalize-charset": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-6.0.2.tgz", + "integrity": "sha512-a8N9czmdnrjPHa3DeFlwqst5eaL5W8jYu3EBbTTkI5FHkfMhFZh1EGbku6jhHhIzTA6tquI2P42NtZ59M/H/kQ==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-display-values": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-display-values/-/postcss-normalize-display-values-6.0.2.tgz", + "integrity": "sha512-8H04Mxsb82ON/aAkPeq8kcBbAtI5Q2a64X/mnRRfPXBq7XeogoQvReqxEfc0B4WPq1KimjezNC8flUtC3Qz6jg==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-positions": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-positions/-/postcss-normalize-positions-6.0.2.tgz", + "integrity": "sha512-/JFzI441OAB9O7VnLA+RtSNZvQ0NCFZDOtp6QPFo1iIyawyXg0YI3CYM9HBy1WvwCRHnPep/BvI1+dGPKoXx/Q==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-repeat-style": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-6.0.2.tgz", + "integrity": "sha512-YdCgsfHkJ2jEXwR4RR3Tm/iOxSfdRt7jplS6XRh9Js9PyCR/aka/FCb6TuHT2U8gQubbm/mPmF6L7FY9d79VwQ==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-string": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-string/-/postcss-normalize-string-6.0.2.tgz", + "integrity": "sha512-vQZIivlxlfqqMp4L9PZsFE4YUkWniziKjQWUtsxUiVsSSPelQydwS8Wwcuw0+83ZjPWNTl02oxlIvXsmmG+CiQ==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-timing-functions": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-6.0.2.tgz", + "integrity": "sha512-a+YrtMox4TBtId/AEwbA03VcJgtyW4dGBizPl7e88cTFULYsprgHWTbfyjSLyHeBcK/Q9JhXkt2ZXiwaVHoMzA==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-unicode": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-unicode/-/postcss-normalize-unicode-6.1.0.tgz", + "integrity": "sha512-QVC5TQHsVj33otj8/JD869Ndr5Xcc/+fwRh4HAsFsAeygQQXm+0PySrKbr/8tkDKzW+EVT3QkqZMfFrGiossDg==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-url": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-6.0.2.tgz", + "integrity": "sha512-kVNcWhCeKAzZ8B4pv/DnrU1wNh458zBNp8dh4y5hhxih5RZQ12QWMuQrDgPRw3LRl8mN9vOVfHl7uhvHYMoXsQ==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-normalize-whitespace": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-whitespace/-/postcss-normalize-whitespace-6.0.2.tgz", + "integrity": "sha512-sXZ2Nj1icbJOKmdjXVT9pnyHQKiSAyuNQHSgRCUgThn2388Y9cGVDR+E9J9iAYbSbLHI+UUwLVl1Wzco/zgv0Q==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-opacity-percentage": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-opacity-percentage/-/postcss-opacity-percentage-3.0.0.tgz", + "integrity": "sha512-K6HGVzyxUxd/VgZdX04DCtdwWJ4NGLG212US4/LA1TLAbHgmAsTWVR86o+gGIbFtnTkfOpb9sCRBx8K7HO66qQ==", + "funding": [ + { + "type": "kofi", + "url": "https://ko-fi.com/mrcgrtz" + }, + { + "type": "liberapay", + "url": "https://liberapay.com/mrcgrtz" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-ordered-values": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-6.0.2.tgz", + "integrity": "sha512-VRZSOB+JU32RsEAQrO94QPkClGPKJEL/Z9PCBImXMhIeK5KAYo6slP/hBYlLgrCjFxyqvn5VC81tycFEDBLG1Q==", + "license": "MIT", + "dependencies": { + "cssnano-utils": "^4.0.2", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-overflow-shorthand": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/postcss-overflow-shorthand/-/postcss-overflow-shorthand-6.0.0.tgz", + "integrity": "sha512-BdDl/AbVkDjoTofzDQnwDdm/Ym6oS9KgmO7Gr+LHYjNWJ6ExORe4+3pcLQsLA9gIROMkiGVjjwZNoL/mpXHd5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-page-break": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/postcss-page-break/-/postcss-page-break-3.0.4.tgz", + "integrity": "sha512-1JGu8oCjVXLa9q9rFTo4MbeeA5FMe00/9C7lN4va606Rdb+HkxXtXsmEDrIraQ11fGz/WvKWa8gMuCKkrXpTsQ==", + "license": "MIT", + "peerDependencies": { + "postcss": "^8" + } + }, + "node_modules/postcss-place": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/postcss-place/-/postcss-place-10.0.0.tgz", + "integrity": "sha512-5EBrMzat2pPAxQNWYavwAfoKfYcTADJ8AXGVPcUZ2UkNloUTWzJQExgrzrDkh3EKzmAx1evfTAzF9I8NGcc+qw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-preset-env": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/postcss-preset-env/-/postcss-preset-env-10.1.1.tgz", + "integrity": "sha512-wqqsnBFD6VIwcHHRbhjTOcOi4qRVlB26RwSr0ordPj7OubRRxdWebv/aLjKLRR8zkZrbxZyuus03nOIgC5elMQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "@csstools/postcss-cascade-layers": "^5.0.1", + "@csstools/postcss-color-function": "^4.0.6", + "@csstools/postcss-color-mix-function": "^3.0.6", + "@csstools/postcss-content-alt-text": "^2.0.4", + "@csstools/postcss-exponential-functions": "^2.0.5", + "@csstools/postcss-font-format-keywords": "^4.0.0", + "@csstools/postcss-gamut-mapping": "^2.0.6", + "@csstools/postcss-gradients-interpolation-method": "^5.0.6", + "@csstools/postcss-hwb-function": "^4.0.6", + "@csstools/postcss-ic-unit": "^4.0.0", + "@csstools/postcss-initial": "^2.0.0", + "@csstools/postcss-is-pseudo-class": "^5.0.1", + "@csstools/postcss-light-dark-function": "^2.0.7", + "@csstools/postcss-logical-float-and-clear": "^3.0.0", + "@csstools/postcss-logical-overflow": "^2.0.0", + "@csstools/postcss-logical-overscroll-behavior": "^2.0.0", + "@csstools/postcss-logical-resize": "^3.0.0", + "@csstools/postcss-logical-viewport-units": "^3.0.3", + "@csstools/postcss-media-minmax": "^2.0.5", + "@csstools/postcss-media-queries-aspect-ratio-number-values": "^3.0.4", + "@csstools/postcss-nested-calc": "^4.0.0", + "@csstools/postcss-normalize-display-values": "^4.0.0", + "@csstools/postcss-oklab-function": "^4.0.6", + "@csstools/postcss-progressive-custom-properties": "^4.0.0", + "@csstools/postcss-random-function": "^1.0.1", + "@csstools/postcss-relative-color-syntax": "^3.0.6", + "@csstools/postcss-scope-pseudo-class": "^4.0.1", + "@csstools/postcss-sign-functions": "^1.1.0", + "@csstools/postcss-stepped-value-functions": "^4.0.5", + "@csstools/postcss-text-decoration-shorthand": "^4.0.1", + "@csstools/postcss-trigonometric-functions": "^4.0.5", + "@csstools/postcss-unset-value": "^4.0.0", + "autoprefixer": "^10.4.19", + "browserslist": "^4.23.1", + "css-blank-pseudo": "^7.0.1", + "css-has-pseudo": "^7.0.1", + "css-prefers-color-scheme": "^10.0.0", + "cssdb": "^8.2.1", + "postcss-attribute-case-insensitive": "^7.0.1", + "postcss-clamp": "^4.1.0", + "postcss-color-functional-notation": "^7.0.6", + "postcss-color-hex-alpha": "^10.0.0", + "postcss-color-rebeccapurple": "^10.0.0", + "postcss-custom-media": "^11.0.5", + "postcss-custom-properties": "^14.0.4", + "postcss-custom-selectors": "^8.0.4", + "postcss-dir-pseudo-class": "^9.0.1", + "postcss-double-position-gradients": "^6.0.0", + "postcss-focus-visible": "^10.0.1", + "postcss-focus-within": "^9.0.1", + "postcss-font-variant": "^5.0.0", + "postcss-gap-properties": "^6.0.0", + "postcss-image-set-function": "^7.0.0", + "postcss-lab-function": "^7.0.6", + "postcss-logical": "^8.0.0", + "postcss-nesting": "^13.0.1", + "postcss-opacity-percentage": "^3.0.0", + "postcss-overflow-shorthand": "^6.0.0", + "postcss-page-break": "^3.0.4", + "postcss-place": "^10.0.0", + "postcss-pseudo-class-any-link": "^10.0.1", + "postcss-replace-overflow-wrap": "^4.0.0", + "postcss-selector-not": "^8.0.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-pseudo-class-any-link": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/postcss-pseudo-class-any-link/-/postcss-pseudo-class-any-link-10.0.1.tgz", + "integrity": "sha512-3el9rXlBOqTFaMFkWDOkHUTQekFIYnaQY55Rsp8As8QQkpiSgIYEcF/6Ond93oHiDsGb4kad8zjt+NPlOC1H0Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-pseudo-class-any-link/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-reduce-idents": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/postcss-reduce-idents/-/postcss-reduce-idents-6.0.3.tgz", + "integrity": "sha512-G3yCqZDpsNPoQgbDUy3T0E6hqOQ5xigUtBQyrmq3tn2GxlyiL0yyl7H+T8ulQR6kOcHJ9t7/9H4/R2tv8tJbMA==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-reduce-initial": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-reduce-initial/-/postcss-reduce-initial-6.1.0.tgz", + "integrity": "sha512-RarLgBK/CrL1qZags04oKbVbrrVK2wcxhvta3GCxrZO4zveibqbRPmm2VI8sSgCXwoUHEliRSbOfpR0b/VIoiw==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "caniuse-api": "^3.0.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-reduce-transforms": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-6.0.2.tgz", + "integrity": "sha512-sB+Ya++3Xj1WaT9+5LOOdirAxP7dJZms3GRcYheSPi1PiTMigsxHAdkrbItHxwYHr4kt1zL7mmcHstgMYT+aiA==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-replace-overflow-wrap": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-replace-overflow-wrap/-/postcss-replace-overflow-wrap-4.0.0.tgz", + "integrity": "sha512-KmF7SBPphT4gPPcKZc7aDkweHiKEEO8cla/GjcBK+ckKxiZslIu3C4GCRW3DNfL0o7yW7kMQu9xlZ1kXRXLXtw==", + "license": "MIT", + "peerDependencies": { + "postcss": "^8.0.3" + } + }, + "node_modules/postcss-selector-not": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/postcss-selector-not/-/postcss-selector-not-8.0.1.tgz", + "integrity": "sha512-kmVy/5PYVb2UOhy0+LqUYAhKj7DUGDpSWa5LZqlkWJaaAV+dxxsOG3+St0yNLu6vsKD7Dmqx+nWQt0iil89+WA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "postcss": "^8.4" + } + }, + "node_modules/postcss-selector-not/node_modules/postcss-selector-parser": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.0.0.tgz", + "integrity": "sha512-9RbEr1Y7FFfptd/1eEdntyjMwLeghW1bHX9GWjXo19vx4ytPQhANltvVxDggzJl7mnWM+dX28kb6cyS/4iQjlQ==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz", + "integrity": "sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-sort-media-queries": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/postcss-sort-media-queries/-/postcss-sort-media-queries-5.2.0.tgz", + "integrity": "sha512-AZ5fDMLD8SldlAYlvi8NIqo0+Z8xnXU2ia0jxmuhxAU+Lqt9K+AlmLNJ/zWEnE9x+Zx3qL3+1K20ATgNOr3fAA==", + "license": "MIT", + "dependencies": { + "sort-css-media-queries": "2.2.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "postcss": "^8.4.23" + } + }, + "node_modules/postcss-svgo": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-6.0.3.tgz", + "integrity": "sha512-dlrahRmxP22bX6iKEjOM+c8/1p+81asjKT+V5lrgOH944ryx/OHpclnIbGsKVd3uWOXFLYJwCVf0eEkJGvO96g==", + "license": "MIT", + "dependencies": { + "postcss-value-parser": "^4.2.0", + "svgo": "^3.2.0" + }, + "engines": { + "node": "^14 || ^16 || >= 18" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-unique-selectors": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-6.0.4.tgz", + "integrity": "sha512-K38OCaIrO8+PzpArzkLKB42dSARtC2tmG6PvD4b1o1Q2E9Os8jzfWFfSy/rixsHwohtsDdFtAWGjFVFUdwYaMg==", + "license": "MIT", + "dependencies": { + "postcss-selector-parser": "^6.0.16" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "license": "MIT" + }, + "node_modules/postcss-zindex": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-6.0.2.tgz", + "integrity": "sha512-5BxW9l1evPB/4ZIc+2GobEBoKC+h8gPGCMi+jxsYvd2x0mjq7wazk6DrP71pStqxE9Foxh5TVnonbWpFZzXaYg==", + "license": "MIT", + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/preact": { + "version": "10.24.3", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.24.3.tgz", + "integrity": "sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, + "node_modules/pretty-error": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-4.0.0.tgz", + "integrity": "sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.20", + "renderkid": "^3.0.0" + } + }, + "node_modules/pretty-time": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pretty-time/-/pretty-time-1.1.0.tgz", + "integrity": "sha512-28iF6xPQrP8Oa6uxE6a1biz+lWeTOAPKggvjB8HAs6nVMKZwf5bG++632Dx614hIWgUPkgivRfG+a8uAXGTIbA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/prism-react-renderer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.1.tgz", + "integrity": "sha512-ey8Ls/+Di31eqzUxC46h8MksNuGx/n0AAC8uKpwFau4RPDYLuE3EXTp8N8G2vX2N7UC/+IXeNUnlWBGGcAG+Ig==", + "license": "MIT", + "dependencies": { + "@types/prismjs": "^1.26.0", + "clsx": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.0.0" + } + }, + "node_modules/prismjs": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", + "integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/property-information": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.5.0.tgz", + "integrity": "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "license": "ISC" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pupa": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-3.1.0.tgz", + "integrity": "sha512-FLpr4flz5xZTSJxSeaheeMKN/EDzMdK7b8PTOC6a5PYFKTucWbdqjgqaEyH0shFiSJrVB1+Qqi4Tk19ccU6Aug==", + "license": "MIT", + "dependencies": { + "escape-goat": "^4.0.0" + }, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-fast-compare": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.2.tgz", + "integrity": "sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==", + "license": "MIT" + }, + "node_modules/react-helmet-async": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/react-helmet-async/-/react-helmet-async-1.3.0.tgz", + "integrity": "sha512-9jZ57/dAn9t3q6hneQS0wukqC2ENOBgMNVEhb/ZG9ZSxUetzVIw4iAmEU38IaVg3QGYauQPhSeUTuIUtFglWpg==", + "license": "Apache-2.0", + "dependencies": { + "@babel/runtime": "^7.12.5", + "invariant": "^2.2.4", + "prop-types": "^15.7.2", + "react-fast-compare": "^3.2.0", + "shallowequal": "^1.1.0" + }, + "peerDependencies": { + "react": "^16.6.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^16.6.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/react-json-view-lite": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/react-json-view-lite/-/react-json-view-lite-2.4.1.tgz", + "integrity": "sha512-fwFYknRIBxjbFm0kBDrzgBy1xa5tDg2LyXXBepC5f1b+MY3BUClMCsvanMPn089JbV1Eg3nZcrp0VCuH43aXnA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0" + } + }, + "node_modules/react-loadable": { + "name": "@docusaurus/react-loadable", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-6.0.0.tgz", + "integrity": "sha512-YMMxTUQV/QFSnbgrP3tjDzLHRg7vsbMn8e9HAa8o/1iXoiomo48b7sk/kkmWEuWNDPJVlKSJRB6Y2fHqdJk+SQ==", + "license": "MIT", + "dependencies": { + "@types/react": "*" + }, + "peerDependencies": { + "react": "*" + } + }, + "node_modules/react-loadable-ssr-addon-v5-slorber": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/react-loadable-ssr-addon-v5-slorber/-/react-loadable-ssr-addon-v5-slorber-1.0.1.tgz", + "integrity": "sha512-lq3Lyw1lGku8zUEJPDxsNm1AfYHBrO9Y1+olAYwpUJ2IGFBskM0DMKok97A6LWUpHm+o7IvQBOWu9MLenp9Z+A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.3" + }, + "engines": { + "node": ">=10.13.0" + }, + "peerDependencies": { + "react-loadable": "*", + "webpack": ">=4.41.1 || 5.x" + } + }, + "node_modules/react-router": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz", + "integrity": "sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.13", + "history": "^4.9.0", + "hoist-non-react-statics": "^3.1.0", + "loose-envify": "^1.3.1", + "path-to-regexp": "^1.7.0", + "prop-types": "^15.6.2", + "react-is": "^16.6.0", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0" + }, + "peerDependencies": { + "react": ">=15" + } + }, + "node_modules/react-router-config": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/react-router-config/-/react-router-config-5.1.1.tgz", + "integrity": "sha512-DuanZjaD8mQp1ppHjgnnUnyOlqYXZVjnov/JzFhjLEwd3Z4dYjMSnqrEzzGThH47vpCOqPPwJM2FtthLeJ8Pbg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.1.2" + }, + "peerDependencies": { + "react": ">=15", + "react-router": ">=5" + } + }, + "node_modules/react-router-dom": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.3.4.tgz", + "integrity": "sha512-m4EqFMHv/Ih4kpcBCONHbkT68KoAeHN4p3lAGoNryfHi0dMy0kCzEZakiKRsvg5wHZ/JLrLW8o8KomWiz/qbYQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.13", + "history": "^4.9.0", + "loose-envify": "^1.3.1", + "prop-types": "^15.6.2", + "react-router": "5.3.4", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0" + }, + "peerDependencies": { + "react": ">=15" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/recma-build-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-build-jsx/-/recma-build-jsx-1.0.0.tgz", + "integrity": "sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-build-jsx": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-jsx/-/recma-jsx-1.0.0.tgz", + "integrity": "sha512-5vwkv65qWwYxg+Atz95acp8DMu1JDSqdGkA2Of1j6rCreyFUE/gp15fC8MnGEuG1W68UKjM6x6+YTWIh7hZM/Q==", + "license": "MIT", + "dependencies": { + "acorn-jsx": "^5.0.0", + "estree-util-to-js": "^2.0.0", + "recma-parse": "^1.0.0", + "recma-stringify": "^1.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-parse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-parse/-/recma-parse-1.0.0.tgz", + "integrity": "sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "esast-util-from-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/recma-stringify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/recma-stringify/-/recma-stringify-1.0.0.tgz", + "integrity": "sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "estree-util-to-js": "^2.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "license": "MIT" + }, + "node_modules/regenerate-unicode-properties": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.2.0.tgz", + "integrity": "sha512-DqHn3DwbmmPVzeKj9woBadqmXxLvQoQIwu7nopMc72ztvxVmVk2SBhSnx67zuye5TP+lJsb/TBQsjLKhnDf3MA==", + "license": "MIT", + "dependencies": { + "regenerate": "^1.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, + "node_modules/regenerator-transform": { + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.2.tgz", + "integrity": "sha512-hfMp2BoF0qOk3uc5V20ALGDS2ddjQaLrdl7xrGXvAIow7qeWRM2VA2HuCHkUKk9slq3VwEwLNK3DFBqDfPGYtg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.8.4" + } + }, + "node_modules/regexpu-core": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-6.1.1.tgz", + "integrity": "sha512-k67Nb9jvwJcJmVpw0jPttR1/zVfnKf8Km0IPatrU/zJ5XeG3+Slx0xLXs9HByJSzXzrlz5EDvN6yLNMDc2qdnw==", + "license": "MIT", + "dependencies": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.2.0", + "regjsgen": "^0.8.0", + "regjsparser": "^0.11.0", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/registry-auth-token": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-5.0.2.tgz", + "integrity": "sha512-o/3ikDxtXaA59BmZuZrJZDJv8NMDGSj+6j6XaeBmHw8eY1i1qd9+6H+LjVvQXx3HN6aRCGa1cUdJ9RaJZUugnQ==", + "license": "MIT", + "dependencies": { + "@pnpm/npm-conf": "^2.1.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/registry-url": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-6.0.1.tgz", + "integrity": "sha512-+crtS5QjFRqFCoQmvGduwYWEBng99ZvmFvF+cUJkGYF1L1BfU8C6Zp9T7f5vPAwyLkUExpvK+ANVZmGU49qi4Q==", + "license": "MIT", + "dependencies": { + "rc": "1.2.8" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/regjsgen": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.8.0.tgz", + "integrity": "sha512-RvwtGe3d7LvWiDQXeQw8p5asZUmfU1G/l6WbUXeHta7Y2PEIvBTwH6E2EfmYUK8pxcxEdEmaomqyp0vZZ7C+3Q==", + "license": "MIT" + }, + "node_modules/regjsparser": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.11.2.tgz", + "integrity": "sha512-3OGZZ4HoLJkkAZx/48mTXJNlmqTGOzc0o9OWQPuWpkOlXXPbyN6OafCcoXUnBqE2D3f/T5L+pWc1kdEmnfnRsA==", + "license": "BSD-2-Clause", + "dependencies": { + "jsesc": "~3.0.2" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, + "node_modules/rehype-raw": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", + "integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-raw": "^9.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-recma": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rehype-recma/-/rehype-recma-1.0.0.tgz", + "integrity": "sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "hast-util-to-estree": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/relateurl": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", + "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/remark-directive": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/remark-directive/-/remark-directive-3.0.0.tgz", + "integrity": "sha512-l1UyWJ6Eg1VPU7Hm/9tt0zKtReJQNOA4+iDMAxTyZNWnJnFlbS/7zhiel/rogTLQ2vMYwDzSJa4BiVNqGlqIMA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-directive": "^3.0.0", + "micromark-extension-directive": "^3.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-emoji": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/remark-emoji/-/remark-emoji-4.0.1.tgz", + "integrity": "sha512-fHdvsTR1dHkWKev9eNyhTo4EFwbUvJ8ka9SgeWkMPYFX4WoI7ViVBms3PjlQYgw5TLvNQso3GUB/b/8t3yo+dg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.2", + "emoticon": "^4.0.1", + "mdast-util-find-and-replace": "^3.0.1", + "node-emoji": "^2.1.0", + "unified": "^11.0.4" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/remark-frontmatter": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/remark-frontmatter/-/remark-frontmatter-5.0.0.tgz", + "integrity": "sha512-XTFYvNASMe5iPN0719nPrdItC9aU0ssC4v14mH1BCi1u0n1gAocqcujWUrByftZTbLhRtiKRyjYTSIOcr69UVQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-frontmatter": "^2.0.0", + "micromark-extension-frontmatter": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-gfm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.0.tgz", + "integrity": "sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-gfm": "^3.0.0", + "micromark-extension-gfm": "^3.0.0", + "remark-parse": "^11.0.0", + "remark-stringify": "^11.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-mdx": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-3.1.0.tgz", + "integrity": "sha512-Ngl/H3YXyBV9RcRNdlYsZujAmhsxwzxpDzpDEhFBVAGthS4GDgnctpDjgFl/ULx5UEDzqtW1cyBSNKqYYrqLBA==", + "license": "MIT", + "dependencies": { + "mdast-util-mdx": "^3.0.0", + "micromark-extension-mdxjs": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.1.tgz", + "integrity": "sha512-g/osARvjkBXb6Wo0XvAeXQohVta8i84ACbenPpoSsxTOQH/Ae0/RGP4WZgnMH5pMLpsj4FG7OHmcIcXxpza8eQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-stringify": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-11.0.0.tgz", + "integrity": "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-to-markdown": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/renderkid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz", + "integrity": "sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==", + "license": "MIT", + "dependencies": { + "css-select": "^4.1.3", + "dom-converter": "^0.2.0", + "htmlparser2": "^6.1.0", + "lodash": "^4.17.21", + "strip-ansi": "^6.0.1" + } + }, + "node_modules/renderkid/node_modules/css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/renderkid/node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/renderkid/node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/renderkid/node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/renderkid/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/renderkid/node_modules/htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-like": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", + "integrity": "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==", + "engines": { + "node": "*" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "license": "MIT", + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", + "license": "MIT" + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-pathname": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz", + "integrity": "sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==", + "license": "MIT" + }, + "node_modules/responselike": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz", + "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==", + "license": "MIT", + "dependencies": { + "lowercase-keys": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", + "license": "Unlicense" + }, + "node_modules/roughjs": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", + "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", + "license": "MIT", + "dependencies": { + "hachure-fill": "^0.5.2", + "path-data-parser": "^0.1.0", + "points-on-curve": "^0.2.0", + "points-on-path": "^0.2.1" + } + }, + "node_modules/rtlcss": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/rtlcss/-/rtlcss-4.3.0.tgz", + "integrity": "sha512-FI+pHEn7Wc4NqKXMXFM+VAYKEj/mRIcW4h24YVwVtyjI+EqGrLc2Hx/Ny0lrZ21cBWU2goLy36eqMcNj3AQJig==", + "license": "MIT", + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0", + "postcss": "^8.4.21", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "rtlcss": "bin/rtlcss.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "license": "ISC" + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/schema-dts": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/schema-dts/-/schema-dts-1.1.5.tgz", + "integrity": "sha512-RJr9EaCmsLzBX2NDiO5Z3ux2BVosNZN5jo0gWgsyKvxKIUL5R3swNvoorulAeL9kLB0iTSX7V6aokhla2m7xbg==", + "license": "Apache-2.0" + }, + "node_modules/schema-utils": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.2.0.tgz", + "integrity": "sha512-L0jRsrPpjdckP3oPug3/VxNKt2trR8TcabrM6FOAAlvC/9Phcmm+cuAgTlxBqdBR1WJx7Naj9WHw+aOmheSVbw==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/search-insights": { + "version": "2.17.3", + "resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz", + "integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==", + "license": "MIT", + "peer": true + }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", + "license": "MIT" + }, + "node_modules/selfsigned": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz", + "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "license": "MIT", + "dependencies": { + "@types/node-forge": "^1.3.0", + "node-forge": "^1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-4.0.0.tgz", + "integrity": "sha512-0Ju4+6A8iOnpL/Thra7dZsSlOHYAHIeMxfhWQRI1/VLcT3WDBZKKtQt/QkBOsiIN9ZpuvHE6cGZ0x4glCMmfiA==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serialize-javascript": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "license": "BSD-3-Clause", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/seroval": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.1.1.tgz", + "integrity": "sha512-rqEO6FZk8mv7Hyv4UCj3FD3b6Waqft605TLfsCe/BiaylRpyyMC0b+uA5TJKawX3KzMrdi3wsLbCaLplrQmBvQ==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/seroval-plugins": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.1.1.tgz", + "integrity": "sha512-qNSy1+nUj7hsCOon7AO4wdAIo9P0jrzAMp18XhiOzA6/uO5TKtP7ScozVJ8T293oRIvi5wyCHSM4TrJo/c/GJA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "seroval": "^1.0" + } + }, + "node_modules/serve-handler": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.6.tgz", + "integrity": "sha512-x5RL9Y2p5+Sh3D38Fh9i/iQ5ZK+e4xuXRd/pGbM4D13tgo/MGwbttUk8emytcr1YYzBYs+apnUngBDFYfpjPuQ==", + "license": "MIT", + "dependencies": { + "bytes": "3.0.0", + "content-disposition": "0.5.2", + "mime-types": "2.1.18", + "minimatch": "3.1.2", + "path-is-inside": "1.0.2", + "path-to-regexp": "3.3.0", + "range-parser": "1.2.0" + } + }, + "node_modules/serve-handler/node_modules/path-to-regexp": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.3.0.tgz", + "integrity": "sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw==", + "license": "MIT" + }, + "node_modules/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-index/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/serve-index/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "license": "MIT", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "license": "ISC" + }, + "node_modules/serve-index/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/serve-index/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "license": "ISC" + }, + "node_modules/serve-index/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shallowequal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", + "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==", + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "license": "MIT", + "dependencies": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/sirv": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", + "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "license": "MIT", + "dependencies": { + "@polka/url": "^1.0.0-next.24", + "mrmime": "^2.0.0", + "totalist": "^3.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/sitemap": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-7.1.2.tgz", + "integrity": "sha512-ARCqzHJ0p4gWt+j7NlU5eDlIO9+Rkr/JhPFZKKQ1l5GCus7rJH4UdrlVAh0xC/gDS/Qir2UMxqYNHtsKr2rpCw==", + "license": "MIT", + "dependencies": { + "@types/node": "^17.0.5", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.2.4" + }, + "bin": { + "sitemap": "dist/cli.js" + }, + "engines": { + "node": ">=12.0.0", + "npm": ">=5.6.0" + } + }, + "node_modules/sitemap/node_modules/@types/node": { + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", + "license": "MIT" + }, + "node_modules/skin-tone": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/skin-tone/-/skin-tone-2.0.0.tgz", + "integrity": "sha512-kUMbT1oBJCpgrnKoSr0o6wPtvRWT9W9UKvGLwfJYO2WuahZRHOpEyL1ckyMGgMWh0UdpmaoFqKKD29WTomNEGA==", + "license": "MIT", + "dependencies": { + "unicode-emoji-modifier-base": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/snake-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.4.tgz", + "integrity": "sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==", + "license": "MIT", + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/sockjs": { + "version": "0.3.24", + "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", + "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", + "license": "MIT", + "dependencies": { + "faye-websocket": "^0.11.3", + "uuid": "^8.3.2", + "websocket-driver": "^0.7.4" + } + }, + "node_modules/sockjs/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/solid-js": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.3.tgz", + "integrity": "sha512-5ba3taPoZGt9GY3YlsCB24kCg0Lv/rie/HTD4kG6h4daZZz7+yK02xn8Vx8dLYBc9i6Ps5JwAbEiqjmKaLB3Ag==", + "license": "MIT", + "dependencies": { + "csstype": "^3.1.0", + "seroval": "^1.1.0", + "seroval-plugins": "^1.1.0" + } + }, + "node_modules/sort-css-media-queries": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/sort-css-media-queries/-/sort-css-media-queries-2.2.0.tgz", + "integrity": "sha512-0xtkGhWCC9MGt/EzgnvbbbKhqWjl1+/rncmhTh5qCpbYguXh6S/qwePfv/JQ8jePXXmqingylxoC49pCkSPIbA==", + "license": "MIT", + "engines": { + "node": ">= 6.3.0" + } + }, + "node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 8" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/srcset": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/srcset/-/srcset-4.0.0.tgz", + "integrity": "sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/std-env": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.8.0.tgz", + "integrity": "sha512-Bc3YwwCB+OzldMxOXJIIvC6cPRWr/LxOp48CdQTOkPyk/t4JWWJbrilwBd7RJzKV8QW7tJkcgAmeuLLJugl5/w==", + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/stringify-object": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", + "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==", + "license": "BSD-2-Clause", + "dependencies": { + "get-own-enumerable-property-symbols": "^3.0.0", + "is-obj": "^1.0.1", + "is-regexp": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/style-to-object": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.8.tgz", + "integrity": "sha512-xT47I/Eo0rwJmaXC4oilDGDWLohVhR6o/xAQcPQN8q6QBuZVL8qMYL85kLmST5cPjAorwvqIA4qXTRQoYHaL6g==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.4" + } + }, + "node_modules/stylehacks": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/stylehacks/-/stylehacks-6.1.1.tgz", + "integrity": "sha512-gSTTEQ670cJNoaeIp9KX6lZmm8LJ3jPB5yJmX8Zq/wQxOsAFXV3qjWzHas3YYk1qesuVIyYWWUpZ0vSE/dTSGg==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.23.0", + "postcss-selector-parser": "^6.0.16" + }, + "engines": { + "node": "^14 || ^16 || >=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/stylis": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.3.6.tgz", + "integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/svg-parser": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz", + "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==", + "license": "MIT" + }, + "node_modules/svgo": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.2.tgz", + "integrity": "sha512-OoohrmuUlBs8B8o6MB2Aevn+pRIH9zDALSR+6hhqVfa6fRwG/Qw9VUMSMW9VNg2CFc/MTIfabtdOVl9ODIJjpw==", + "license": "MIT", + "dependencies": { + "@trysound/sax": "0.2.0", + "commander": "^7.2.0", + "css-select": "^5.1.0", + "css-tree": "^2.3.1", + "css-what": "^6.1.0", + "csso": "^5.0.5", + "picocolors": "^1.0.0" + }, + "bin": { + "svgo": "bin/svgo" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/svgo" + } + }, + "node_modules/svgo/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/terser": { + "version": "5.36.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.36.0.tgz", + "integrity": "sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==", + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.10", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz", + "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==", + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.20", + "jest-worker": "^27.4.5", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.1", + "terser": "^5.26.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/terser-webpack-plugin/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/terser-webpack-plugin/node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/terser-webpack-plugin/node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/terser-webpack-plugin/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, + "node_modules/terser-webpack-plugin/node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/terser-webpack-plugin/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/terser/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, + "node_modules/thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", + "license": "MIT" + }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", + "license": "MIT" + }, + "node_modules/tiny-warning": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", + "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.1.tgz", + "integrity": "sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ==", + "license": "MIT" + }, + "node_modules/tinypool": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.0.2.tgz", + "integrity": "sha512-al6n+QEANGFOMf/dmUMsuS5/r9B06uwlyNjZZql/zv8J7ybHCgoihBNORZCY2mzUuAnomQa2JdhyHKzZxPCrFA==", + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/totalist": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", + "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/ts-dedent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", + "integrity": "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==", + "license": "MIT", + "engines": { + "node": ">=6.10" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "license": "MIT", + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/ufo": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.5.4.tgz", + "integrity": "sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "license": "MIT" + }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz", + "integrity": "sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-emoji-modifier-base": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unicode-emoji-modifier-base/-/unicode-emoji-modifier-base-1.0.0.tgz", + "integrity": "sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "license": "MIT", + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.2.0.tgz", + "integrity": "sha512-4IehN3V/+kkr5YeSSDDQG8QLqO26XpL2XP3GQtqwlT/QYSECAwFztxVHjlbh0+gjJ3XmNLS0zDsbgs9jWKExLg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz", + "integrity": "sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unique-string": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-3.0.0.tgz", + "integrity": "sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==", + "license": "MIT", + "dependencies": { + "crypto-random-string": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", + "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", + "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.0" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/update-notifier": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-6.0.2.tgz", + "integrity": "sha512-EDxhTEVPZZRLWYcJ4ZXjGFN0oP7qYvbXWzEgRm/Yql4dHX5wDbvh89YHP6PK1lzZJYrMtXUuZZz8XGK+U6U1og==", + "license": "BSD-2-Clause", + "dependencies": { + "boxen": "^7.0.0", + "chalk": "^5.0.1", + "configstore": "^6.0.0", + "has-yarn": "^3.0.0", + "import-lazy": "^4.0.0", + "is-ci": "^3.0.1", + "is-installed-globally": "^0.4.0", + "is-npm": "^6.0.0", + "is-yarn-global": "^0.4.0", + "latest-version": "^7.0.0", + "pupa": "^3.1.0", + "semver": "^7.3.7", + "semver-diff": "^4.0.0", + "xdg-basedir": "^5.1.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/yeoman/update-notifier?sponsor=1" + } + }, + "node_modules/update-notifier/node_modules/boxen": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.1.1.tgz", + "integrity": "sha512-2hCgjEmP8YLWQ130n2FerGv7rYpfBmnmp9Uy2Le1vge6X3gZIfSmEzP5QTDElFxcvVcXlEn8Aq6MU/PZygIOog==", + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.1", + "camelcase": "^7.0.1", + "chalk": "^5.2.0", + "cli-boxes": "^3.0.0", + "string-width": "^5.1.2", + "type-fest": "^2.13.0", + "widest-line": "^4.0.1", + "wrap-ansi": "^8.1.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/update-notifier/node_modules/camelcase": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-7.0.1.tgz", + "integrity": "sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/update-notifier/node_modules/chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/url-loader": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-4.1.1.tgz", + "integrity": "sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==", + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.0", + "mime-types": "^2.1.27", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "file-loader": "*", + "webpack": "^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "file-loader": { + "optional": true + } + } + }, + "node_modules/url-loader/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/url-loader/node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/url-loader/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, + "node_modules/url-loader/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/url-loader/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/url-loader/node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/utila": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/utila/-/utila-0.4.0.tgz", + "integrity": "sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==", + "license": "MIT" + }, + "node_modules/utility-types": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.11.0.tgz", + "integrity": "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/value-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/value-equal/-/value-equal-1.0.1.tgz", + "integrity": "sha512-NOJ6JZCAWr0zlxZt+xqCHNTEKOsrks2HQd4MqhP1qy4z1SkbEP467eNx6TgDKXMvUOb+OENfJCZwM+16n7fRfw==", + "license": "MIT" + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-location": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz", + "integrity": "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", + "license": "MIT", + "dependencies": { + "vscode-languageserver-protocol": "3.17.5" + }, + "bin": { + "installServerIntoExtension": "bin/installServerIntoExtension" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "license": "MIT", + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.12.tgz", + "integrity": "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==", + "license": "MIT" + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "license": "MIT" + }, + "node_modules/vscode-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", + "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", + "license": "MIT" + }, + "node_modules/watchpack": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz", + "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==", + "license": "MIT", + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "license": "MIT", + "dependencies": { + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/webpack": { + "version": "5.96.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.96.1.tgz", + "integrity": "sha512-l2LlBSvVZGhL4ZrPwyr8+37AunkcYj5qh8o6u2/2rzoPc8gxFJkLj1WxNgooi9pnoc06jh0BjuXnamM4qlujZA==", + "license": "MIT", + "dependencies": { + "@types/eslint-scope": "^3.7.7", + "@types/estree": "^1.0.6", + "@webassemblyjs/ast": "^1.12.1", + "@webassemblyjs/wasm-edit": "^1.12.1", + "@webassemblyjs/wasm-parser": "^1.12.1", + "acorn": "^8.14.0", + "browserslist": "^4.24.0", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.17.1", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^3.2.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.3.10", + "watchpack": "^2.4.1", + "webpack-sources": "^3.2.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-bundle-analyzer": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.10.2.tgz", + "integrity": "sha512-vJptkMm9pk5si4Bv922ZbKLV8UTT4zib4FPgXMhgzUny0bfDDkLXAVQs3ly3fS4/TN9ROFtb0NFrm04UXFE/Vw==", + "license": "MIT", + "dependencies": { + "@discoveryjs/json-ext": "0.5.7", + "acorn": "^8.0.4", + "acorn-walk": "^8.0.0", + "commander": "^7.2.0", + "debounce": "^1.2.1", + "escape-string-regexp": "^4.0.0", + "gzip-size": "^6.0.0", + "html-escaper": "^2.0.2", + "opener": "^1.5.2", + "picocolors": "^1.0.0", + "sirv": "^2.0.3", + "ws": "^7.3.1" + }, + "bin": { + "webpack-bundle-analyzer": "lib/bin/analyzer.js" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/webpack-dev-middleware": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", + "license": "MIT", + "dependencies": { + "colorette": "^2.0.10", + "memfs": "^3.4.3", + "mime-types": "^2.1.31", + "range-parser": "^1.2.1", + "schema-utils": "^4.0.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/webpack-dev-middleware/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack-dev-middleware/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack-dev-middleware/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack-dev-server": { + "version": "4.15.2", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.15.2.tgz", + "integrity": "sha512-0XavAZbNJ5sDrCbkpWL8mia0o5WPOd2YGtxrEiZkBK9FjLppIUK2TgxK6qGD2P3hUXTJNNPVibrerKcx5WkR1g==", + "license": "MIT", + "dependencies": { + "@types/bonjour": "^3.5.9", + "@types/connect-history-api-fallback": "^1.3.5", + "@types/express": "^4.17.13", + "@types/serve-index": "^1.9.1", + "@types/serve-static": "^1.13.10", + "@types/sockjs": "^0.3.33", + "@types/ws": "^8.5.5", + "ansi-html-community": "^0.0.8", + "bonjour-service": "^1.0.11", + "chokidar": "^3.5.3", + "colorette": "^2.0.10", + "compression": "^1.7.4", + "connect-history-api-fallback": "^2.0.0", + "default-gateway": "^6.0.3", + "express": "^4.17.3", + "graceful-fs": "^4.2.6", + "html-entities": "^2.3.2", + "http-proxy-middleware": "^2.0.3", + "ipaddr.js": "^2.0.1", + "launch-editor": "^2.6.0", + "open": "^8.0.9", + "p-retry": "^4.5.0", + "rimraf": "^3.0.2", + "schema-utils": "^4.0.0", + "selfsigned": "^2.1.1", + "serve-index": "^1.9.1", + "sockjs": "^0.3.24", + "spdy": "^4.0.2", + "webpack-dev-middleware": "^5.3.4", + "ws": "^8.13.0" + }, + "bin": { + "webpack-dev-server": "bin/webpack-dev-server.js" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.37.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + }, + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-dev-server/node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/webpack-merge": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", + "license": "MIT", + "dependencies": { + "clone-deep": "^4.0.1", + "flat": "^5.0.2", + "wildcard": "^2.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/webpack-sources": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/webpack/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/webpack/node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "license": "MIT", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/webpack/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "license": "MIT" + }, + "node_modules/webpack/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack/node_modules/schema-utils": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz", + "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/webpackbar": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpackbar/-/webpackbar-6.0.1.tgz", + "integrity": "sha512-TnErZpmuKdwWBdMoexjio3KKX6ZtoKHRVvLIU0A47R0VVBDtx3ZyOJDktgYixhoJokZTYTt1Z37OkO9pnGJa9Q==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^4.3.2", + "chalk": "^4.1.2", + "consola": "^3.2.3", + "figures": "^3.2.0", + "markdown-table": "^2.0.0", + "pretty-time": "^1.1.0", + "std-env": "^3.7.0", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=14.21.3" + }, + "peerDependencies": { + "webpack": "3 || 4 || 5" + } + }, + "node_modules/webpackbar/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/webpackbar/node_modules/markdown-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", + "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==", + "license": "MIT", + "dependencies": { + "repeat-string": "^1.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/webpackbar/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/webpackbar/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/websocket-driver": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", + "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", + "license": "Apache-2.0", + "dependencies": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/widest-line": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-4.0.1.tgz", + "integrity": "sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==", + "license": "MIT", + "dependencies": { + "string-width": "^5.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wildcard": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz", + "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "node_modules/ws": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xdg-basedir": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-5.1.0.tgz", + "integrity": "sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "license": "MIT", + "dependencies": { + "sax": "^1.2.4" + }, + "bin": { + "xml-js": "bin/cli.js" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", + "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/docs/package.json b/docs/package.json index 1e4020dd4..d44749548 100644 --- a/docs/package.json +++ b/docs/package.json @@ -14,18 +14,26 @@ "write-heading-ids": "docusaurus write-heading-ids" }, "dependencies": { - "@cmfcmf/docusaurus-search-local": "^1.1.0", - "@docusaurus/core": "^2.2.0", - "@docusaurus/module-type-aliases": "^2.2.0", - "@docusaurus/plugin-google-gtag": "^2.4.1", - "@docusaurus/preset-classic": "^2.4.1", - "@docusaurus/theme-mermaid": "^2.4.1", - "@mdx-js/react": "^1.6.22", - "asciinema-player": "^3.5.0", - "clsx": "^1.2.1", - "prism-react-renderer": "^2.0.6", - "react": "^17.0.2", - "react-dom": "^17.0.2" + "@cmfcmf/docusaurus-search-local": "1.2.0", + "@docusaurus/core": "3.8.0", + "@docusaurus/preset-classic": "3.8.0", + "@docusaurus/theme-mermaid": "3.8.0", + "@mdx-js/react": "3.1.0", + "asciinema-player": "3.10.0", + "clsx": "2.1.1", + "prism-react-renderer": "2.4.1", + "react": "18.3.1", + "react-dom": "18.3.1" + }, + "devDependencies": { + "@docusaurus/module-type-aliases": "3.8.0", + "@docusaurus/types": "3.8.0" + }, + "overrides": { + "@cmfcmf/docusaurus-search-local": { + "@docusaurus/core": "3.8.0", + "cheerio": "1.0.0-rc.12" + } }, "browserslist": { "production": [ diff --git a/docs/screencasts/docker/Dockerfile b/docs/screencasts/docker/Dockerfile index 836c83c65..d0b0a5a91 100644 --- a/docs/screencasts/docker/Dockerfile +++ b/docs/screencasts/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04@sha256:2b7412e6465c3c7fc5bb21d3e6f1917c167358449fecac8176c6e496e5c1f05f +FROM ubuntu:24.04@sha256:b59d21599a2b151e23eea5f6602f4af4d7d31c4e236d22bf0b62b86d2e386b8f # Install requirements RUN apt-get update && apt-get install -y software-properties-common &&\ diff --git a/docs/screencasts/docker/create-cluster.expect b/docs/screencasts/docker/create-cluster.expect index a9b290892..6476b4ec9 100755 --- a/docs/screencasts/docker/create-cluster.expect +++ b/docs/screencasts/docker/create-cluster.expect @@ -23,26 +23,21 @@ spawn asciinema rec --overwrite /recordings/create-cluster.cast send "\r" expect_prompt -run_command "# Step 1: Create cloud environment" +run_command "# Step 1: Create the Constellation cluster" expect_prompt -run_command "constellation create" +run_command "constellation apply" expect -re "y\/n" send "y" send "\r" expect_prompt -run_command "# Step 2: Initialize Constellation" -expect_prompt -run_command "constellation apply" -expect_prompt - run_command "# Wait for cluster to finish bootstrapping..." expect_prompt # Without a sleep we only see a single node, not 5. run_command "sleep 300" expect_prompt -run_command "# Step 3: Connect to Constellation" +run_command "# Step 2: Connect to Constellation" expect_prompt run_command "export KUBECONFIG=/constellation/constellation-admin.conf" expect_prompt diff --git a/docs/screencasts/docker/github-readme.expect b/docs/screencasts/docker/github-readme.expect index d5399c54b..6c096647e 100644 --- a/docs/screencasts/docker/github-readme.expect +++ b/docs/screencasts/docker/github-readme.expect @@ -23,13 +23,12 @@ spawn asciinema rec --overwrite /recordings/github-readme.cast send "\r" expect_prompt -run_command "constellation create" -expect -re "y\/n" -send "y" -send "\r" +run_command "constellation config generate gcp" +expect_prompt +run_command "constellation iam create gcp --update-config --projectID constellation-331613 --serviceAccountID constellation-demo --zone europe-west3-b -y" expect_prompt -run_command "constellation apply" +run_command "constellation apply -y" expect_prompt run_command "export KUBECONFIG=/constellation/constellation-admin.conf" diff --git a/docs/screencasts/generate-readme-svg.sh b/docs/screencasts/generate-readme-svg.sh index dcb44745f..fb6756890 100755 --- a/docs/screencasts/generate-readme-svg.sh +++ b/docs/screencasts/generate-readme-svg.sh @@ -8,13 +8,6 @@ # screenrecordings container. A full script run takes ~20min. # -# Create IAM configuration -docker run -it \ - -v "${HOME}"/.config/gcloud:/root/.config/gcloud \ - -v "$(pwd)"/recordings:/recordings \ - -v "$(pwd)"/constellation:/constellation \ - screenrecodings /scripts/configure-cluster.expect - docker build -t screenrecodings docker # Create cast @@ -34,7 +27,7 @@ tail -n+2 recordings/github-readme.cast >> new_header.cast # Then render cast into svg using: # https://github.com/nbedos/termtosvg -termtosvg render new_header.cast readme.svg -t window-frame.svg -D 1ms +python3 -m termtosvg render new_header.cast readme.svg -t window-frame.svg -D 1ms # Copy and cleanup cp readme.svg ../static/img/shell-windowframe.svg diff --git a/docs/sidebars.js b/docs/sidebars.js index 0fa18bd97..21d4ef42e 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -55,6 +55,11 @@ const sidebars = { label: 'Performance', link: { type: 'doc', id: 'overview/performance/performance' }, items: [ + { + type: 'doc', + label: 'Compute benchmarks', + id: 'overview/performance/compute', + }, { type: 'doc', label: 'I/O benchmarks', @@ -96,6 +101,11 @@ const sidebars = { label: 'First steps (local)', id: 'getting-started/first-steps-local', }, + { + type: 'doc', + label: 'Cloud Marketplaces', + id: 'getting-started/marketplaces', + }, { type: 'category', label: 'Examples', @@ -210,6 +220,11 @@ const sidebars = { label: 'Consume SBOMs', id: 'workflows/sbom', }, + { + type: 'doc', + label: 'Reproduce release artifacts', + id: 'workflows/reproducible-builds', + }, { type: 'doc', label: 'Troubleshooting', diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index f502dd906..b9cd96aed 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -7,7 +7,7 @@ /** * Fonts */ -@import url('https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap'); +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap'); /* You can override the default Infima variables here. */ :root { --ifm-color-primary: #8B04DD; @@ -17,7 +17,7 @@ --ifm-color-primary-light: #8B04DD; --ifm-color-primary-lighter: #B873F4; --ifm-color-primary-lightest: #E3D2FF; - --ifm-font-family-base: 'Roboto', sans-serif; + --ifm-font-family-base: 'Inter', sans-serif; --ifm-code-font-size: 95%; /* --ifm-footer-background-color: black; --ifm-footer-link-color: white; diff --git a/docs/src/theme/MDXComponents.js b/docs/src/theme/MDXComponents.js index a0852811a..2a1413d73 100644 --- a/docs/src/theme/MDXComponents.js +++ b/docs/src/theme/MDXComponents.js @@ -10,7 +10,7 @@ export default { ...MDXComponents, // Map the "highlight" tag to our component! // `Highlight` will receive all props that were passed to `highlight` in MDX - tabs: Tabs, - tabItem: TabItem, - asciinemaWidget: AsciinemaWidget, + Tabs, + TabItem, + AsciinemaWidget, }; diff --git a/docs/static/assets/check-sbom.cast b/docs/static/assets/check-sbom.cast index 84ed88d1b..cc846f776 100644 --- a/docs/static/assets/check-sbom.cast +++ b/docs/static/assets/check-sbom.cast @@ -1,988 +1,991 @@ -{"version": 2, "width": 126, "height": 61, "timestamp": 1700561274, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} -[0.008932, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[0.009577, "o", "#"] -[0.141264, "o", " "] -[0.191459, "o", "S"] -[0.241561, "o", "t"] -[0.312989, "o", "e"] -[0.363097, "o", "p"] -[0.413291, "o", " "] -[0.533056, "o", "1"] -[0.58315, "o", ":"] -[0.633265, "o", " "] -[0.683526, "o", "I"] -[0.790435, "o", "n"] -[0.840601, "o", "s"] -[0.922925, "o", "t"] -[0.973095, "o", "a"] -[1.02309, "o", "l"] -[1.191177, "o", "l"] -[1.241231, "o", " "] -[1.330529, "o", "S"] -[1.380693, "o", "L"] -[1.504659, "o", "S"] -[1.554945, "o", "A"] -[1.605005, "o", " "] -[1.711071, "o", "v"] -[1.761134, "o", "e"] -[1.811213, "o", "r"] -[1.861331, "o", "i"] -[1.911491, "o", "f"] -[1.961633, "o", "i"] -[2.011665, "o", "e"] -[2.112902, "o", "r\r\n\u001b[?2004l\r"] -[2.113414, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[3.11389, "o", "c"] -[3.163967, "o", "u"] -[3.214046, "o", "r"] -[3.264265, "o", "l"] -[3.31433, "o", " "] -[3.364549, "o", "-"] -[3.500493, "o", "s"] -[3.557701, "o", "L"] -[3.622905, "o", "O"] -[3.672891, "o", " "] -[3.73915, "o", "h"] -[3.78907, "o", "t"] -[3.876297, "o", "t"] -[3.926362, "o", "p"] -[3.976469, "o", "s"] -[4.02655, "o", ":"] -[4.077031, "o", "/"] -[4.182886, "o", "/"] -[4.232835, "o", "g"] -[4.283122, "o", "i"] -[4.363386, "o", "t"] -[4.464415, "o", "h"] -[4.514589, "o", "u"] -[4.570632, "o", "b"] -[4.620805, "o", "."] -[4.702861, "o", "c"] -[4.877016, "o", "o"] -[4.92702, "o", "m"] -[4.977068, "o", "/"] -[5.027206, "o", "s"] -[5.15044, "o", "l"] -[5.200532, "o", "s"] -[5.441607, "o", "a"] -[5.49182, "o", "-"] -[5.549908, "o", "f"] -[5.621859, "o", "r"] -[5.672041, "o", "a"] -[5.722136, "o", "m"] -[5.772209, "o", "e"] -[5.85533, "o", "w"] -[5.905375, "o", "o"] -[6.063516, "o", "r"] -[6.113633, "o", "k"] -[6.163773, "o", "/"] -[6.213904, "o", "s"] -[6.263987, "o", "l"] -[6.319162, "o", "s"] -[6.369228, "o", "a"] -[6.41927, "o", "-"] -[6.579462, "o", "v"] -[6.629619, "o", "e"] -[6.81674, "o", "r"] -[6.903818, "o", "i"] -[6.953896, "o", "f"] -[7.004042, "o", "i"] -[7.054041, "o", "e"] -[7.104181, "o", "r"] -[7.154257, "o", "/"] -[7.204423, "o", "r"] -[7.254532, "o", "e"] -[7.375607, "o", "l"] -[7.425695, "o", "e"] -[7.475893, "o", "a"] -[7.530899, "o", "s"] -[7.580918, "o", "e"] -[7.682216, "o", "s"] -[7.732235, "o", "/"] -[7.782339, "o", "l"] -[7.832531, "o", "a"] -[7.885794, "o", "t"] -[7.960755, "o", "e"] -[8.012758, "o", "s"] -[8.062779, "o", "t"] -[8.112972, "o", "/"] -[8.163061, "o", "d"] -[8.216231, "o", "o"] -[8.266214, "o", "w"] -[8.316508, "o", "n"] -[8.385525, "o", "l"] -[8.435887, "o", "o"] -[8.515997, "o", "a"] -[8.565831, "o", "d"] -[8.616203, "o", "/"] -[8.666091, "o", "s"] -[8.716136, "o", "l"] -[8.766216, "o", "s"] -[8.880352, "o", "a"] -[8.930521, "o", "-"] -[9.005502, "o", "v"] -[9.055563, "o", "e"] -[9.105742, "o", "r"] -[9.155921, "o", "i"] -[9.20592, "o", "f"] -[9.268006, "o", "i"] -[9.318239, "o", "e"] -[9.36828, "o", "r"] -[9.4186, "o", "-"] -[9.468548, "o", "l"] -[9.518879, "o", "i"] -[9.568912, "o", "n"] -[9.64293, "o", "u"] -[9.693047, "o", "x"] -[9.743136, "o", "-"] -[9.793074, "o", "a"] -[9.87738, "o", "m"] -[9.927405, "o", "d"] -[10.012581, "o", "6"] -[10.062818, "o", "4\r\n\u001b[?2004l\r"] -[11.859531, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[11.85974, "o", "s"] -[11.910195, "o", "u"] -[12.000091, "o", "d"] -[12.091095, "o", "o"] -[12.141163, "o", " "] -[12.191274, "o", "i"] -[12.241368, "o", "n"] -[12.345518, "o", "s"] -[12.4948, "o", "t"] -[12.544793, "o", "a"] -[12.594872, "o", "l"] -[12.644993, "o", "l"] -[12.695275, "o", " "] -[12.745188, "o", "s"] -[12.810414, "o", "l"] -[12.860481, "o", "s"] -[12.947596, "o", "a"] -[12.997782, "o", "-"] -[13.052894, "o", "v"] -[13.102959, "o", "e"] -[13.298031, "o", "r"] -[13.470203, "o", "i"] -[13.520348, "o", "f"] -[13.570439, "o", "i"] -[13.653724, "o", "e"] -[13.703834, "o", "r"] -[13.754056, "o", "-"] -[13.80401, "o", "l"] -[13.854046, "o", "i"] -[13.904191, "o", "n"] -[13.95428, "o", "u"] -[14.004516, "o", "x"] -[14.054701, "o", "-"] -[14.104722, "o", "a"] -[14.155125, "o", "m"] -[14.205139, "o", "d"] -[14.254999, "o", "6"] -[14.305131, "o", "4"] -[14.355337, "o", " "] -[14.405436, "o", "/"] -[14.455503, "o", "u"] -[14.533638, "o", "s"] -[14.583758, "o", "r"] -[14.634127, "o", "/"] -[14.684033, "o", "l"] -[14.733964, "o", "o"] -[14.784038, "o", "c"] -[14.944247, "o", "a"] -[15.018467, "o", "l"] -[15.068553, "o", "/"] -[15.118861, "o", "b"] -[15.168955, "o", "i"] -[15.226943, "o", "n"] -[15.277085, "o", "/"] -[15.327085, "o", "s"] -[15.377205, "o", "l"] -[15.42733, "o", "s"] -[15.492306, "o", "a"] -[15.542632, "o", "-"] -[15.694616, "o", "v"] -[15.99462, "o", "e"] -[16.119727, "o", "r"] -[16.169788, "o", "i"] -[16.219861, "o", "f"] -[16.269993, "o", "i"] -[16.320001, "o", "e"] -[16.370128, "o", "r\r\n\u001b[?2004l\r"] -[16.430778, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[17.431315, "o", "#"] -[17.48156, "o", " "] -[17.531731, "o", "S"] -[17.581771, "o", "t"] -[17.63194, "o", "e"] -[17.716124, "o", "p"] -[17.766197, "o", " "] -[17.871288, "o", "2"] -[17.92124, "o", ":"] -[17.984469, "o", " "] -[18.117443, "o", "D"] -[18.173713, "o", "o"] -[18.223958, "o", "w"] -[18.277033, "o", "n"] -[18.326991, "o", "l"] -[18.404075, "o", "o"] -[18.454166, "o", "a"] -[18.504277, "o", "d"] -[18.554471, "o", " "] -[18.604696, "o", "C"] -[18.654908, "o", "o"] -[18.704916, "o", "n"] -[18.755223, "o", "s"] -[18.805081, "o", "t"] -[18.855225, "o", "e"] -[18.923272, "o", "l"] -[18.973353, "o", "l"] -[19.02347, "o", "a"] -[19.073548, "o", "t"] -[19.123915, "o", "i"] -[19.190818, "o", "o"] -[19.240945, "o", "n"] -[19.290999, "o", " "] -[19.362158, "o", "S"] -[19.520263, "o", "B"] -[19.597348, "o", "O"] -[19.647542, "o", "M"] -[19.697588, "o", " "] -[19.75178, "o", "a"] -[19.8019, "o", "n"] -[19.851945, "o", "d"] -[19.901987, "o", " "] -[20.072275, "o", "p"] -[20.122251, "o", "r"] -[20.172322, "o", "o"] -[20.222364, "o", "v"] -[20.272541, "o", "e"] -[20.390616, "o", "n"] -[20.440715, "o", "a"] -[20.496891, "o", "n"] -[20.546944, "o", "c"] -[20.597098, "o", "e\r\n\u001b[?2004l\r"] -[20.59719, "o", "\u001b[?2004h"] -[20.59723, "o", "\u001b[38;2;139;4;221m$\u001b[0m "] -[21.597652, "o", "c"] -[21.647954, "o", "u"] -[21.714011, "o", "r"] -[21.76527, "o", "l"] -[21.815203, "o", " "] -[21.888284, "o", "-"] -[21.997539, "o", "s"] -[22.056417, "o", "L"] -[22.106561, "o", "O"] -[22.156753, "o", " "] -[22.206772, "o", "h"] -[22.278051, "o", "t"] -[22.328083, "o", "t"] -[22.525148, "o", "p"] -[22.575268, "o", "s"] -[22.625491, "o", ":"] -[22.675543, "o", "/"] -[22.889619, "o", "/"] -[22.939841, "o", "g"] -[22.989992, "o", "i"] -[23.040097, "o", "t"] -[23.111159, "o", "h"] -[23.161379, "o", "u"] -[23.211518, "o", "b"] -[23.261624, "o", "."] -[23.376899, "o", "c"] -[23.427007, "o", "o"] -[23.477061, "o", "m"] -[23.527347, "o", "/"] -[23.577305, "o", "e"] -[23.627351, "o", "d"] -[23.677474, "o", "g"] -[23.727708, "o", "e"] -[23.777631, "o", "l"] -[23.827787, "o", "e"] -[23.877967, "o", "s"] -[23.928002, "o", "s"] -[23.978061, "o", "s"] -[24.059214, "o", "y"] -[24.109314, "o", "s"] -[24.159561, "o", "/"] -[24.246854, "o", "c"] -[24.296965, "o", "o"] -[24.479063, "o", "n"] -[24.561224, "o", "s"] -[24.631212, "o", "t"] -[24.681374, "o", "e"] -[24.731471, "o", "l"] -[24.781883, "o", "l"] -[24.831782, "o", "a"] -[24.881835, "o", "t"] -[24.980981, "o", "i"] -[25.031166, "o", "o"] -[25.081269, "o", "n"] -[25.131396, "o", "/"] -[25.181444, "o", "r"] -[25.231886, "o", "e"] -[25.281653, "o", "l"] -[25.331908, "o", "e"] -[25.381959, "o", "a"] -[25.432159, "o", "s"] -[25.482247, "o", "e"] -[25.532352, "o", "s"] -[25.582389, "o", "/"] -[25.638499, "o", "l"] -[25.751651, "o", "a"] -[25.801779, "o", "t"] -[25.871037, "o", "e"] -[25.921123, "o", "s"] -[26.018314, "o", "t"] -[26.068447, "o", "/"] -[26.182607, "o", "d"] -[26.239752, "o", "o"] -[26.289855, "o", "w"] -[26.341004, "o", "n"] -[26.391053, "o", "l"] -[26.519114, "o", "o"] -[26.629168, "o", "a"] -[26.679404, "o", "d"] -[26.729374, "o", "/"] -[26.779579, "o", "c"] -[26.82984, "o", "o"] -[26.879851, "o", "n"] -[27.051032, "o", "s"] -[27.123121, "o", "t"] -[27.173225, "o", "e"] -[27.223345, "o", "l"] -[27.339312, "o", "l"] -[27.455576, "o", "a"] -[27.505749, "o", "t"] -[27.555858, "o", "i"] -[27.605936, "o", "o"] -[27.685443, "o", "n"] -[27.735381, "o", "."] -[27.785769, "o", "s"] -[27.835832, "o", "p"] -[27.887014, "o", "d"] -[27.936929, "o", "x"] -[27.987124, "o", "."] -[28.052133, "o", "s"] -[28.10244, "o", "b"] -[28.152423, "o", "o"] -[28.202801, "o", "m\r\n\u001b[?2004l\r"] -[29.080668, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[29.081069, "o", "c"] -[29.131331, "o", "u"] -[29.214492, "o", "r"] -[29.265414, "o", "l"] -[29.315504, "o", " "] -[29.375648, "o", "-"] -[29.448841, "o", "s"] -[29.512119, "o", "L"] -[29.561861, "o", "O"] -[29.611917, "o", " "] -[29.662172, "o", "h"] -[29.712324, "o", "t"] -[29.804493, "o", "t"] -[29.854592, "o", "p"] -[29.911548, "o", "s"] -[29.96179, "o", ":"] -[30.011923, "o", "/"] -[30.061853, "o", "/"] -[30.111979, "o", "g"] -[30.162143, "o", "i"] -[30.217339, "o", "t"] -[30.320338, "o", "h"] -[30.3704, "o", "u"] -[30.493599, "o", "b"] -[30.543653, "o", "."] -[30.6041, "o", "c"] -[30.724868, "o", "o"] -[30.802316, "o", "m"] -[30.852344, "o", "/"] -[30.902496, "o", "e"] -[30.952393, "o", "d"] -[31.002506, "o", "g"] -[31.052705, "o", "e"] -[31.126045, "o", "l"] -[31.179926, "o", "e"] -[31.248084, "o", "s"] -[31.298109, "o", "s"] -[31.348155, "o", "s"] -[31.398256, "o", "y"] -[31.501338, "o", "s"] -[31.551605, "o", "/"] -[31.629828, "o", "c"] -[31.67997, "o", "o"] -[31.730163, "o", "n"] -[31.780021, "o", "s"] -[31.914152, "o", "t"] -[31.964274, "o", "e"] -[32.014508, "o", "l"] -[32.09643, "o", "l"] -[32.318639, "o", "a"] -[32.370963, "o", "t"] -[32.420914, "o", "i"] -[32.470897, "o", "o"] -[32.556047, "o", "n"] -[32.606125, "o", "/"] -[32.664377, "o", "r"] -[32.714388, "o", "e"] -[32.7644, "o", "l"] -[32.814834, "o", "e"] -[32.942748, "o", "a"] -[32.992855, "o", "s"] -[33.046982, "o", "e"] -[33.187068, "o", "s"] -[33.237334, "o", "/"] -[33.287402, "o", "l"] -[33.33738, "o", "a"] -[33.387569, "o", "t"] -[33.442786, "o", "e"] -[33.523927, "o", "s"] -[33.574007, "o", "t"] -[33.624026, "o", "/"] -[33.674158, "o", "d"] -[33.724276, "o", "o"] -[33.802395, "o", "w"] -[33.852514, "o", "n"] -[33.902772, "o", "l"] -[33.966884, "o", "o"] -[34.016882, "o", "a"] -[34.070257, "o", "d"] -[34.120062, "o", "/"] -[34.170175, "o", "c"] -[34.220312, "o", "o"] -[34.296435, "o", "n"] -[34.353536, "o", "s"] -[34.467705, "o", "t"] -[34.620743, "o", "e"] -[34.670933, "o", "l"] -[34.72101, "o", "l"] -[34.771168, "o", "a"] -[34.866231, "o", "t"] -[34.973378, "o", "i"] -[35.023568, "o", "o"] -[35.073646, "o", "n"] -[35.12387, "o", "."] -[35.223133, "o", "i"] -[35.27294, "o", "n"] -[35.323132, "o", "t"] -[35.373057, "o", "o"] -[35.423209, "o", "t"] -[35.473376, "o", "o"] -[35.523546, "o", "."] -[35.573611, "o", "j"] -[35.623616, "o", "s"] -[35.673875, "o", "o"] -[35.723938, "o", "n"] -[35.792034, "o", "l\r\n\u001b[?2004l\r"] -[36.186056, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[37.186513, "o", "#"] -[37.257617, "o", " "] -[37.307901, "o", "S"] -[37.492906, "o", "t"] -[37.543067, "o", "e"] -[37.593153, "o", "p"] -[37.643266, "o", " "] -[37.737387, "o", "3"] -[37.787569, "o", ":"] -[37.850595, "o", " "] -[37.900734, "o", "C"] -[38.028926, "o", "h"] -[38.150115, "o", "e"] -[38.200237, "o", "c"] -[38.274158, "o", "k"] -[38.324354, "o", " "] -[38.410543, "o", "i"] -[38.498527, "o", "n"] -[38.548889, "o", "t"] -[38.598887, "o", "e"] -[38.648986, "o", "g"] -[38.732151, "o", "r"] -[38.782274, "o", "i"] -[38.832394, "o", "t"] -[38.914443, "o", "y"] -[38.964634, "o", " "] -[39.02799, "o", "o"] -[39.078011, "o", "f"] -[39.127858, "o", " "] -[39.19805, "o", "S"] -[39.248251, "o", "B"] -[39.298294, "o", "O"] -[39.348457, "o", "M\r\n\u001b[?2004l\r"] -[39.348628, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[40.349181, "o", "s"] -[40.39929, "o", "l"] -[40.449374, "o", "s"] -[40.499471, "o", "a"] -[40.549568, "o", "-"] -[40.72066, "o", "v"] -[40.770818, "o", "e"] -[40.826959, "o", "r"] -[40.876994, "o", "i"] -[40.92722, "o", "f"] -[40.977336, "o", "i"] -[41.027382, "o", "e"] -[41.104554, "o", "r"] -[41.154647, "o", " "] -[41.220778, "o", "v"] -[41.270915, "o", "e"] -[41.320829, "o", "r"] -[41.37099, "o", "i"] -[41.504054, "o", "f"] -[41.554278, "o", "y"] -[41.604399, "o", "-"] -[41.654446, "o", "a"] -[41.78566, "o", "r"] -[41.899841, "o", "t"] -[41.952997, "o", "i"] -[42.03791, "o", "f"] -[42.10509, "o", "a"] -[42.155336, "o", "c"] -[42.205283, "o", "t"] -[42.255449, "o", " "] -[42.305548, "o", "c"] -[42.418782, "o", "o"] -[42.498816, "o", "n"] -[42.549063, "o", "s"] -[42.599256, "o", "t"] -[42.669208, "o", "e"] -[42.719425, "o", "l"] -[42.76953, "o", "l"] -[42.819835, "o", "a"] -[42.869879, "o", "t"] -[42.919917, "o", "i"] -[42.970221, "o", "o"] -[43.020157, "o", "n"] -[43.070409, "o", "."] -[43.120485, "o", "s"] -[43.170621, "o", "p"] -[43.231781, "o", "d"] -[43.310721, "o", "x"] -[43.36082, "o", "."] -[43.410897, "o", "s"] -[43.468008, "o", "b"] -[43.518172, "o", "o"] -[43.56832, "o", "m"] -[43.618534, "o", " "] -[43.668582, "o", "-"] -[43.743699, "o", "-"] -[43.793805, "o", "p"] -[43.895885, "o", "r"] -[43.961078, "o", "o"] -[44.011142, "o", "v"] -[44.061318, "o", "e"] -[44.233265, "o", "n"] -[44.28351, "o", "a"] -[44.333483, "o", "n"] -[44.383578, "o", "c"] -[44.433703, "o", "e"] -[44.483977, "o", "-"] -[44.534158, "o", "p"] -[44.619236, "o", "a"] -[44.669294, "o", "t"] -[44.75346, "o", "h"] -[44.803586, "o", " "] -[44.853725, "o", "c"] -[44.903778, "o", "o"] -[44.954065, "o", "n"] -[45.009137, "o", "s"] -[45.059114, "o", "t"] -[45.192273, "o", "e"] -[45.242511, "o", "l"] -[45.29252, "o", "l"] -[45.342706, "o", "a"] -[45.392835, "o", "t"] -[45.503853, "o", "i"] -[45.553932, "o", "o"] -[45.60412, "o", "n"] -[45.654282, "o", "."] -[45.704323, "o", "i"] -[45.754358, "o", "n"] -[45.808573, "o", "t"] -[45.875785, "o", "o"] -[46.006639, "o", "t"] -[46.056841, "o", "o"] -[46.106802, "o", "."] -[46.156793, "o", "j"] -[46.206994, "o", "s"] -[46.288179, "o", "o"] -[46.338242, "o", "n"] -[46.388316, "o", "l"] -[46.438484, "o", " "] -[46.552518, "o", "-"] -[46.602935, "o", "-"] -[46.65277, "o", "s"] -[46.702907, "o", "o"] -[46.753036, "o", "u"] -[46.803297, "o", "r"] -[46.853317, "o", "c"] -[46.955531, "o", "e"] -[47.005591, "o", "-"] -[47.149643, "o", "u"] -[47.199739, "o", "r"] -[47.27289, "o", "i"] -[47.322927, "o", " "] -[47.373042, "o", "g"] -[47.423152, "o", "i"] -[47.473237, "o", "t"] -[47.523346, "o", "h"] -[47.573507, "o", "u"] -[47.623529, "o", "b"] -[47.673705, "o", "."] -[47.784823, "o", "c"] -[47.866899, "o", "o"] -[47.916923, "o", "m"] -[47.967037, "o", "/"] -[48.017131, "o", "e \r"] -[48.067224, "o", "d"] -[48.117293, "o", "g"] -[48.214414, "o", "e"] -[48.264477, "o", "l"] -[48.314567, "o", "e"] -[48.364751, "o", "s"] -[48.414818, "o", "s"] -[48.464853, "o", "s"] -[48.568027, "o", "y"] -[48.618002, "o", "s"] -[48.668125, "o", "/"] -[48.749248, "o", "c"] -[48.799436, "o", "o"] -[48.849567, "o", "n"] -[48.899593, "o", "s"] -[48.949562, "o", "t"] -[49.053698, "o", "e"] -[49.111748, "o", "l"] -[49.161775, "o", "l"] -[49.277894, "o", "a"] -[49.327946, "o", "t"] -[49.378067, "o", "i"] -[49.42835, "o", "o"] -[49.478276, "o", "n\r\n\u001b[?2004l\r"] -[50.394465, "o", "Verified signature against tlog entry index 49747663 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77afd31550fc3f5fb69e285d1ebb57f76d52dc7679627197e6cbcfc9d91d8e10a57\r\n"] -[50.404546, "o", "Verified build using builder \"https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.9.0\" at commit 246b9ce069acb0933ea8d2c786383b3e46d26936\r\nVerifying artifact constellation.spdx.sbom: PASSED\r\n\r\nPASSED: Verified SLSA provenance\r\n"] -[50.407053, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[50.407202, "o", "#"] -[50.497577, "o", " "] -[50.547814, "o", "S"] -[50.626846, "o", "t"] -[50.677025, "o", "e"] -[50.782025, "o", "p"] -[50.83215, "o", " "] -[50.882104, "o", "4"] -[50.932268, "o", ":"] -[50.99357, "o", " "] -[51.089648, "o", "I"] -[51.250751, "o", "n"] -[51.3011, "o", "s"] -[51.426982, "o", "t"] -[51.47705, "o", "a"] -[51.527261, "o", "l"] -[51.57736, "o", "l"] -[51.627412, "o", " "] -[51.677791, "o", "g"] -[51.727631, "o", "r"] -[51.777893, "o", "y"] -[51.903054, "o", "p"] -[51.953176, "o", "e"] -[52.00319, "o", " "] -[52.053413, "o", "("] -[52.103482, "o", "s"] -[52.153516, "o", "e"] -[52.203577, "o", "c"] -[52.253617, "o", "u"] -[52.33386, "o", "r"] -[52.383862, "o", "i"] -[52.433938, "o", "t"] -[52.484051, "o", "y"] -[52.534216, "o", " "] -[52.726258, "o", "s"] -[52.800455, "o", "c"] -[52.85059, "o", "a"] -[52.938852, "o", "n"] -[52.988921, "o", "n"] -[53.09114, "o", "e"] -[53.141294, "o", "r"] -[53.191603, "o", ")\r\n\u001b[?2004l\r"] -[53.192012, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[54.192748, "o", "c"] -[54.292022, "o", "u"] -[54.365924, "o", "r"] -[54.429108, "o", "l"] -[54.479234, "o", " "] -[54.529317, "o", "-"] -[54.595437, "o", "s"] -[54.645491, "o", "L"] -[54.761594, "o", "O"] -[54.811814, "o", " "] -[55.010766, "o", "h"] -[55.060945, "o", "t"] -[55.111213, "o", "t"] -[55.161276, "o", "p"] -[55.211373, "o", "s"] -[55.26148, "o", ":"] -[55.357628, "o", "/"] -[55.407609, "o", "/"] -[55.457771, "o", "g"] -[55.507823, "o", "i"] -[55.557893, "o", "t"] -[55.607955, "o", "h"] -[55.658103, "o", "u"] -[55.708197, "o", "b"] -[55.758382, "o", "."] -[55.814489, "o", "c"] -[55.901553, "o", "o"] -[55.951725, "o", "m"] -[56.001982, "o", "/"] -[56.05215, "o", "a"] -[56.165206, "o", "n"] -[56.29337, "o", "c"] -[56.593545, "o", "h"] -[56.643543, "o", "o"] -[56.694865, "o", "r"] -[56.745045, "o", "e"] -[56.79515, "o", "/"] -[56.845314, "o", "g"] -[56.957225, "o", "r"] -[57.012327, "o", "y"] -[57.062651, "o", "p"] -[57.113775, "o", "e"] -[57.163826, "o", "/"] -[57.213931, "o", "r"] -[57.264085, "o", "e"] -[57.354048, "o", "l"] -[57.482313, "o", "e"] -[57.53236, "o", "a"] -[57.582428, "o", "s"] -[57.632574, "o", "e"] -[57.731737, "o", "s"] -[57.781907, "o", "/"] -[57.831956, "o", "d"] -[57.881982, "o", "o"] -[57.970203, "o", "w"] -[58.020194, "o", "n"] -[58.083288, "o", "l"] -[58.1364, "o", "o"] -[58.186478, "o", "a"] -[58.236768, "o", "d"] -[58.286905, "o", "/"] -[58.336995, "o", "v"] -[58.500062, "o", "0"] -[58.549998, "o", "."] -[58.610367, "o", "5"] -[58.660422, "o", "6"] -[58.710635, "o", "."] -[58.769744, "o", "0"] -[58.819744, "o", "/"] -[58.869876, "o", "g"] -[58.920285, "o", "r"] -[58.985213, "o", "y"] -[59.044418, "o", "p"] -[59.145653, "o", "e"] -[59.195906, "o", "_"] -[59.245677, "o", "0"] -[59.295921, "o", "."] -[59.346189, "o", "5"] -[59.445198, "o", "6"] -[59.495315, "o", "."] -[59.586382, "o", "0"] -[59.636428, "o", "_"] -[59.830599, "o", "l"] -[59.880789, "o", "i"] -[59.952801, "o", "n"] -[60.03602, "o", "u"] -[60.155059, "o", "x"] -[60.20512, "o", "_"] -[60.255226, "o", "a"] -[60.305441, "o", "m"] -[60.355496, "o", "d"] -[60.434524, "o", "6"] -[60.506646, "o", "4"] -[60.556865, "o", "."] -[60.607103, "o", "t"] -[60.657056, "o", "a"] -[60.70707, "o", "r"] -[60.757328, "o", "."] -[60.817361, "o", "g"] -[60.874486, "o", "z\r\n\u001b[?2004l\r"] -[62.096874, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[62.097045, "o", "t"] -[62.187347, "o", "a"] -[62.237429, "o", "r"] -[62.287458, "o", " "] -[62.347595, "o", "-"] -[62.397861, "o", "x"] -[62.448021, "o", "v"] -[62.497898, "o", "z"] -[62.548018, "o", "f"] -[62.598006, "o", " "] -[62.648216, "o", "g"] -[62.862519, "o", "r"] -[62.912563, "o", "y"] -[62.962849, "o", "p"] -[63.012796, "o", "e"] -[63.06289, "o", "_"] -[63.112971, "o", "0"] -[63.163068, "o", "."] -[63.214124, "o", "5"] -[63.264281, "o", "6"] -[63.314282, "o", "."] -[63.450589, "o", "0"] -[63.500791, "o", "_"] -[63.5509, "o", "l"] -[63.707837, "o", "i"] -[63.757996, "o", "n"] -[63.846101, "o", "u"] -[63.896364, "o", "x"] -[63.946295, "o", "_"] -[63.996403, "o", "a"] -[64.046509, "o", "m"] -[64.154771, "o", "d"] -[64.204873, "o", "6"] -[64.255099, "o", "4"] -[64.305068, "o", "."] -[64.397433, "o", "t"] -[64.447423, "o", "a"] -[64.608657, "o", "r"] -[64.658687, "o", "."] -[64.731963, "o", "g"] -[64.830977, "o", "z\r\n\u001b[?2004l\r"] -[64.837196, "o", "CHANGELOG.md\r\n"] -[64.837301, "o", "LICENSE\r\nREADME.md\r\n"] -[64.837376, "o", "grype\r\n"] -[65.224901, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[66.225285, "o", "s"] -[66.275374, "o", "u"] -[66.325423, "o", "d"] -[66.42752, "o", "o"] -[66.477546, "o", " "] -[66.527842, "o", "i"] -[66.727895, "o", "n"] -[66.777918, "o", "s"] -[66.869087, "o", "t"] -[66.919324, "o", "a"] -[66.969445, "o", "l"] -[67.019703, "o", "l"] -[67.06981, "o", " "] -[67.165927, "o", "g"] -[67.215336, "o", "r"] -[67.265429, "o", "y"] -[67.322694, "o", "p"] -[67.37279, "o", "e"] -[67.422783, "o", " "] -[67.635078, "o", "/"] -[67.701951, "o", "u"] -[67.764182, "o", "s"] -[67.814255, "o", "r"] -[67.864349, "o", "/"] -[67.914559, "o", "l"] -[68.083603, "o", "o"] -[68.138873, "o", "c"] -[68.188858, "o", "a"] -[68.25888, "o", "l"] -[68.308951, "o", "/"] -[68.381143, "o", "b"] -[68.431103, "o", "i"] -[68.497254, "o", "n"] -[68.547305, "o", "/"] -[68.718412, "o", "g"] -[68.76857, "o", "r"] -[68.818586, "o", "y"] -[68.967719, "o", "p"] -[69.017905, "o", "e\r\n\u001b[?2004l\r"] -[69.048126, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[70.048536, "o", "g"] -[70.098537, "o", "r"] -[70.148627, "o", "y"] -[70.198779, "o", "p"] -[70.248791, "o", "e"] -[70.298879, "o", " "] -[70.350092, "o", "-"] -[70.400093, "o", "-"] -[70.515339, "o", "h"] -[70.597288, "o", "e"] -[70.647497, "o", "l"] -[70.756734, "o", "p\r\n\u001b[?2004l\r"] -[70.785235, "o", "A vulnerability scanner for container images, filesystems, and SBOMs.\r\n\r\nSupports the following image sources:\r\n grype yourrepo/yourimage:tag defaults to using images from a Docker daemon\r\n grype path/to/yourproject a Docker tar, OCI tar, OCI directory, SIF container, or generic filesystem directory\r\n grype attestation.json --key cosign.pub extract and scan SBOM from attestation file\r\n\r\nYou can also explicitly specify the scheme to use:\r\n grype podman:yourrepo/yourimage:tag explicitly use the Podman daemon\r\n grype docker:yourrepo/yourimage:tag explicitly use the Docker daemon\r\n grype docker-archive:path/to/yourimage.tar use a tarball from disk for archives created from \"docker save\"\r\n grype oci-archive:path/to/yourimage.tar use a tarball from disk for OCI archives (from Podman or otherwise)\r\n grype oci-dir:path/to/yourimage read directly from a path on disk for OCI layout directories (from Skopeo or otherwise)\r\n grype"] -[70.785322, "o", " singularity:path/to/yourimage.sif read directly from a Singularity Image Format (SIF) container on disk\r\n grype dir:path/to/yourproject read directly from a path on disk (any directory)\r\n grype sbom:path/to/syft.json read Syft JSON from path on disk\r\n grype registry:yourrepo/yourimage:tag pull image directly from a registry (no container runtime required)\r\n grype att:attestation.json --key cosign.pub explicitly use the input as an attestation\r\n grype purl:path/to/purl/file read a newline separated file of purls from a path on disk\r\n\r\nYou can also pipe in Syft JSON directly:\r\n\tsyft yourimage:tag -o json | grype\r\n\r\n"] -[70.785636, "o", "Usage:\r\n grype [IMAGE] [flags]\r\n grype [command]\r\n\r\nAvailable Commands:\r\n completion Generate a shell completion for Grype (listing local docker images)\r\n db vulnerability database operations\r\n help Help about any command\r\n version show the version\r\n\r\nFlags:\r\n --add-cpes-if-none generate CPEs for packages with no CPE data\r\n --by-cve orient results by CVE instead of the original vulnerability ID when possible\r\n -c, --config string application config file\r\n --distro string distro to match against in the format: :\r\n --exclude stringArray exclude paths from being scanned using a glob expression\r\n -f, --fail-on string set the return code to 1 if a vulnerability is found with a severity >= the given severity, options=[negligible low medium high critical]\r\n --file string file to write the report output to (default is STDOUT)\r\n -h, --help help for grype\r\n --key string "] -[70.785738, "o", " File path to a public key to validate attestation\r\n --only-fixed ignore matches for vulnerabilities that are not fixed\r\n --only-notfixed ignore matches for vulnerabilities that are fixed\r\n -o, --output string report output formatter, formats=[json table cyclonedx cyclonedx-json sarif template], deprecated formats=[embedded-cyclonedx-vex-json embedded-cyclonedx-vex-xml]\r\n --platform string an optional platform specifier for container image sources (e.g. 'linux/arm64', 'linux/arm64/v8', 'arm64', 'linux')\r\n -q, --quiet suppress all logging output\r\n -s, --scope string selection of layers to analyze, options=[Squashed AllLayers] (default \"Squashed\")\r\n --show-suppressed show suppressed/ignored vulnerabilities in the output (only supported with table output format)\r\n -t, --template string specify the path to a Go template file (requires 'template' output to be selected)\r\n -v, --verbose count increase verbo"] -[70.785829, "o", "sity (-v = info, -vv = debug)\r\n\r\nUse \"grype [command] --help\" for more information about a command.\r\n"] -[70.787441, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[71.788092, "o", "#"] -[71.89013, "o", " "] -[71.940268, "o", "S"] -[71.990326, "o", "t"] -[72.040558, "o", "e"] -[72.228664, "o", "p"] -[72.27875, "o", " "] -[72.423876, "o", "5"] -[72.47403, "o", ":"] -[72.524158, "o", " "] -[72.586224, "o", "C"] -[72.636353, "o", "h"] -[72.704483, "o", "e"] -[72.771472, "o", "c"] -[72.821835, "o", "k"] -[72.871896, "o", " "] -[72.937137, "o", "f"] -[72.987104, "o", "o"] -[73.259213, "o", "r"] -[73.309314, "o", " "] -[73.35949, "o", "v"] -[73.409802, "o", "u"] -[73.547793, "o", "l"] -[73.597825, "o", "n"] -[73.647998, "o", "e"] -[73.698058, "o", "r"] -[73.849124, "o", "a"] -[73.899343, "o", "b"] -[74.030338, "o", "i"] -[74.080524, "o", "l"] -[74.130738, "o", "i"] -[74.180627, "o", "t"] -[74.249898, "o", "i"] -[74.372851, "o", "e"] -[74.423123, "o", "s\r\n\u001b[?2004l\r"] -[74.423357, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[75.424015, "o", "g"] -[75.535013, "o", "r"] -[75.585127, "o", "y"] -[75.65233, "o", "p"] -[75.702365, "o", "e"] -[75.752464, "o", " "] -[75.802512, "o", "c"] -[76.057608, "o", "o"] -[76.190857, "o", "n"] -[76.355035, "o", "s"] -[76.405008, "o", "t"] -[76.490188, "o", "e"] -[76.549103, "o", "l"] -[76.599392, "o", "l"] -[76.649571, "o", "a"] -[76.699683, "o", "t"] -[76.749804, "o", "i"] -[76.895022, "o", "o"] -[77.058162, "o", "n"] -[77.108178, "o", "."] -[77.158295, "o", "s"] -[77.20836, "o", "p"] -[77.258506, "o", "d"] -[77.33171, "o", "x"] -[77.381758, "o", "."] -[77.442967, "o", "s"] -[77.499071, "o", "b"] -[77.549037, "o", "o"] -[77.5993, "o", "m"] -[77.649348, "o", " "] -[77.699439, "o", "-"] -[77.749703, "o", "o"] -[77.799825, "o", " "] -[77.849839, "o", "t"] -[77.900112, "o", "a"] -[77.952288, "o", "b"] -[78.023502, "o", "l"] -[78.254438, "o", "e"] -[78.304396, "o", " "] -[78.362816, "o", "-"] -[78.426836, "o", "q\r\n\u001b[?2004l\r"] -[93.590379, "o", "No vulnerabilities found\r\n"] -[93.593777, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[93.593943, "o", "exit"] +{"version": 2, "width": 199, "height": 20, "timestamp": 1703674651, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} +[0.003646, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[0.004175, "o", "#"] +[0.136015, "o", " "] +[0.18609, "o", "S"] +[0.236253, "o", "t"] +[0.307376, "o", "e"] +[0.357476, "o", "p"] +[0.407556, "o", " "] +[0.527658, "o", "1"] +[0.577704, "o", ":"] +[0.627909, "o", " "] +[0.678042, "o", "I"] +[0.785138, "o", "n"] +[0.835217, "o", "s"] +[0.917326, "o", "t"] +[0.967415, "o", "a"] +[1.01753, "o", "l"] +[1.18565, "o", "l"] +[1.235784, "o", " "] +[1.324879, "o", "S"] +[1.374955, "o", "L"] +[1.499051, "o", "S"] +[1.549163, "o", "A"] +[1.599267, "o", " "] +[1.705415, "o", "v"] +[1.755487, "o", "e"] +[1.805614, "o", "r"] +[1.855715, "o", "i"] +[1.905798, "o", "f"] +[1.955904, "o", "i"] +[2.006058, "o", "e"] +[2.107178, "o", "r\r\n\u001b[?2004l\r"] +[2.107287, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[3.107803, "o", "c"] +[3.157819, "o", "u"] +[3.207896, "o", "r"] +[3.258013, "o", "l"] +[3.308122, "o", " "] +[3.358213, "o", "-"] +[3.494326, "o", "s"] +[3.551422, "o", "L"] +[3.616539, "o", "O"] +[3.66665, "o", " "] +[3.732749, "o", "h"] +[3.78292, "o", "t"] +[3.869985, "o", "t"] +[3.920077, "o", "p"] +[3.970174, "o", "s"] +[4.020279, "o", ":"] +[4.07039, "o", "/"] +[4.176518, "o", "/"] +[4.226636, "o", "g"] +[4.27675, "o", "i"] +[4.356833, "o", "t"] +[4.457963, "o", "h"] +[4.508026, "o", "u"] +[4.564136, "o", "b"] +[4.614194, "o", "."] +[4.696297, "o", "c"] +[4.870439, "o", "o"] +[4.920538, "o", "m"] +[4.970605, "o", "/"] +[5.020765, "o", "s"] +[5.143868, "o", "l"] +[5.193961, "o", "s"] +[5.435081, "o", "a"] +[5.485189, "o", "-"] +[5.543212, "o", "f"] +[5.615385, "o", "r"] +[5.665492, "o", "a"] +[5.715578, "o", "m"] +[5.765676, "o", "e"] +[5.848808, "o", "w"] +[5.898913, "o", "o"] +[6.057058, "o", "r"] +[6.107124, "o", "k"] +[6.157206, "o", "/"] +[6.207317, "o", "s"] +[6.257429, "o", "l"] +[6.312547, "o", "s"] +[6.362627, "o", "a"] +[6.412767, "o", "-"] +[6.572853, "o", "v"] +[6.622963, "o", "e"] +[6.810128, "o", "r"] +[6.897207, "o", "i"] +[6.947302, "o", "f"] +[6.997406, "o", "i"] +[7.047498, "o", "e"] +[7.097607, "o", "r"] +[7.147711, "o", "/"] +[7.197809, "o", "r"] +[7.247939, "o", "e"] +[7.369049, "o", "l"] +[7.419142, "o", "e"] +[7.469249, "o", "a"] +[7.524363, "o", "s"] +[7.57443, "o", "e"] +[7.675544, "o", "s"] +[7.725661, "o", "/"] +[7.775764, "o", "l"] +[7.825871, "o", "a"] +[7.878951, "o", "t"] +[7.954065, "o", "e"] +[8.006063, "o", "s"] +[8.056225, "o", "t"] +[8.106326, "o", "/"] +[8.156447, "o", "d"] +[8.209553, "o", "o"] +[8.259659, "o", "w"] +[8.309772, "o", "n"] +[8.378873, "o", "l"] +[8.428974, "o", "o"] +[8.509057, "o", "a"] +[8.559148, "o", "d"] +[8.609259, "o", "/"] +[8.659361, "o", "s"] +[8.709457, "o", "l"] +[8.759577, "o", "s"] +[8.873687, "o", "a"] +[8.923784, "o", "-"] +[8.998909, "o", "v"] +[9.049011, "o", "e"] +[9.099123, "o", "r"] +[9.149226, "o", "i"] +[9.199315, "o", "f"] +[9.261418, "o", "i"] +[9.311538, "o", "e"] +[9.361627, "o", "r"] +[9.411721, "o", "-"] +[9.461827, "o", "l"] +[9.511909, "o", "i"] +[9.561995, "o", "n"] +[9.63615, "o", "u"] +[9.686234, "o", "x"] +[9.736357, "o", "-"] +[9.786471, "o", "a"] +[9.870594, "o", "m"] +[9.920695, "o", "d"] +[10.005794, "o", "6"] +[10.05591, "o", "4\r\n\u001b[?2004l\r"] +[10.698194, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[10.698422, "o", "s"] +[10.748578, "o", "u"] +[10.838679, "o", "d"] +[10.929795, "o", "o"] +[10.979865, "o", " "] +[11.029973, "o", "i"] +[11.080037, "o", "n"] +[11.184193, "o", "s"] +[11.333311, "o", "t"] +[11.383376, "o", "a"] +[11.433501, "o", "l"] +[11.483603, "o", "l"] +[11.533693, "o", " "] +[11.583813, "o", "s"] +[11.648921, "o", "l"] +[11.699005, "o", "s"] +[11.786082, "o", "a"] +[11.836227, "o", "-"] +[11.891313, "o", "v"] +[11.941407, "o", "e"] +[12.136563, "o", "r"] +[12.308691, "o", "i"] +[12.358745, "o", "f"] +[12.40883, "o", "i"] +[12.491954, "o", "e"] +[12.542021, "o", "r"] +[12.592149, "o", "-"] +[12.642226, "o", "l"] +[12.692354, "o", "i"] +[12.742446, "o", "n"] +[12.792553, "o", "u"] +[12.842698, "o", "x"] +[12.892761, "o", "-"] +[12.94289, "o", "a"] +[12.992959, "o", "m"] +[13.043064, "o", "d"] +[13.093196, "o", "6"] +[13.143304, "o", "4"] +[13.193394, "o", " "] +[13.243513, "o", "/"] +[13.293596, "o", "u"] +[13.371722, "o", "s"] +[13.421829, "o", "r"] +[13.471913, "o", "/"] +[13.522018, "o", "l"] +[13.572136, "o", "o"] +[13.622208, "o", "c"] +[13.782337, "o", "a"] +[13.856446, "o", "l"] +[13.906541, "o", "/"] +[13.956629, "o", "b"] +[14.006756, "o", "i"] +[14.064865, "o", "n"] +[14.114996, "o", "/"] +[14.165093, "o", "s"] +[14.215206, "o", "l"] +[14.265311, "o", "s"] +[14.330415, "o", "a"] +[14.380486, "o", "-"] +[14.532627, "o", "v"] +[14.832776, "o", "e"] +[14.957839, "o", "r"] +[15.00794, "o", "i"] +[15.058029, "o", "f"] +[15.108138, "o", "i"] +[15.158246, "o", "e"] +[15.20835, "o", "r\r\n\u001b[?2004l\r"] +[15.233551, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[16.234026, "o", "#"] +[16.284076, "o", " "] +[16.334203, "o", "S"] +[16.384295, "o", "t"] +[16.434388, "o", "e"] +[16.51851, "o", "p"] +[16.568597, "o", " "] +[16.673728, "o", "2"] +[16.723817, "o", ":"] +[16.786873, "o", " "] +[16.920009, "o", "D"] +[16.976137, "o", "o"] +[17.026203, "o", "w"] +[17.079303, "o", "n"] +[17.129405, "o", "l"] +[17.206457, "o", "o"] +[17.256599, "o", "a"] +[17.306703, "o", "d"] +[17.356818, "o", " "] +[17.406942, "o", "C"] +[17.457031, "o", "o"] +[17.507127, "o", "n"] +[17.557239, "o", "s"] +[17.607313, "o", "t"] +[17.657453, "o", "e"] +[17.725546, "o", "l"] +[17.77564, "o", "l"] +[17.825738, "o", "a"] +[17.875811, "o", "t"] +[17.925937, "o", "i"] +[17.993053, "o", "o"] +[18.043216, "o", "n"] +[18.093297, "o", " "] +[18.164393, "o", "S"] +[18.322549, "o", "B"] +[18.399612, "o", "O"] +[18.449686, "o", "M"] +[18.499808, "o", " "] +[18.553929, "o", "a"] +[18.604017, "o", "n"] +[18.654099, "o", "d"] +[18.704209, "o", " "] +[18.874334, "o", "p"] +[18.924416, "o", "r"] +[18.9745, "o", "o"] +[19.024648, "o", "v"] +[19.074721, "o", "e"] +[19.19279, "o", "n"] +[19.242929, "o", "a"] +[19.299029, "o", "n"] +[19.349129, "o", "c"] +[19.399267, "o", "e\r\n\u001b[?2004l\r"] +[19.399369, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[20.399761, "o", "c"] +[20.449853, "o", "u"] +[20.515967, "o", "r"] +[20.567067, "o", "l"] +[20.617169, "o", " "] +[20.690326, "o", "-"] +[20.799426, "o", "s"] +[20.858519, "o", "L"] +[20.908641, "o", "O"] +[20.958734, "o", " "] +[21.008841, "o", "h"] +[21.079945, "o", "t"] +[21.130034, "o", "t"] +[21.327195, "o", "p"] +[21.377232, "o", "s"] +[21.427366, "o", ":"] +[21.477465, "o", "/"] +[21.691607, "o", "/"] +[21.741667, "o", "g"] +[21.791786, "o", "i"] +[21.841887, "o", "t"] +[21.913003, "o", "h"] +[21.963089, "o", "u"] +[22.013206, "o", "b"] +[22.063282, "o", "."] +[22.178436, "o", "c"] +[22.228483, "o", "o"] +[22.278612, "o", "m"] +[22.328722, "o", "/"] +[22.37881, "o", "e"] +[22.428906, "o", "d"] +[22.479023, "o", "g"] +[22.529112, "o", "e"] +[22.579226, "o", "l"] +[22.629373, "o", "e"] +[22.67946, "o", "s"] +[22.729553, "o", "s"] +[22.779644, "o", "s"] +[22.860791, "o", "y"] +[22.910863, "o", "s"] +[22.960972, "o", "/"] +[23.048098, "o", "c"] +[23.098182, "o", "o"] +[23.280314, "o", "n"] +[23.362459, "o", "s"] +[23.432532, "o", "t"] +[23.482634, "o", "e"] +[23.53276, "o", "l"] +[23.582816, "o", "l"] +[23.632932, "o", "a"] +[23.683034, "o", "t"] +[23.782135, "o", "i"] +[23.832245, "o", "o"] +[23.882367, "o", "n"] +[23.932454, "o", "/"] +[23.982542, "o", "r"] +[24.032664, "o", "e"] +[24.082759, "o", "l"] +[24.13289, "o", "e"] +[24.182973, "o", "a"] +[24.233055, "o", "s"] +[24.283188, "o", "e"] +[24.333273, "o", "s"] +[24.383414, "o", "/"] +[24.439524, "o", "l"] +[24.552574, "o", "a"] +[24.602663, "o", "t"] +[24.671789, "o", "e"] +[24.721869, "o", "s"] +[24.818959, "o", "t"] +[24.869038, "o", "/"] +[24.983177, "o", "d"] +[25.04024, "o", "o"] +[25.090339, "o", "w"] +[25.141452, "o", "n"] +[25.191591, "o", "l"] +[25.319654, "o", "o"] +[25.42976, "o", "a"] +[25.479847, "o", "d"] +[25.529955, "o", "/"] +[25.580113, "o", "c"] +[25.630194, "o", "o"] +[25.680301, "o", "n"] +[25.851459, "o", "s"] +[25.923535, "o", "t"] +[25.973643, "o", "e"] +[26.023731, "o", "l"] +[26.139862, "o", "l"] +[26.255982, "o", "a"] +[26.30608, "o", "t"] +[26.356198, "o", "i"] +[26.406295, "o", "o"] +[26.485361, "o", "n"] +[26.535497, "o", "."] +[26.58558, "o", "s"] +[26.635716, "o", "p"] +[26.686784, "o", "d"] +[26.736886, "o", "x"] +[26.786998, "o", "."] +[26.852126, "o", "s"] +[26.902199, "o", "b"] +[26.952273, "o", "o"] +[27.002437, "o", "m\r\n\u001b[?2004l\r"] +[27.810738, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[27.811023, "o", "c"] +[27.861134, "o", "u"] +[27.944264, "o", "r"] +[27.995355, "o", "l"] +[28.04542, "o", " "] +[28.10558, "o", "-"] +[28.178674, "o", "s"] +[28.241807, "o", "L"] +[28.291899, "o", "O"] +[28.341988, "o", " "] +[28.392118, "o", "h"] +[28.44221, "o", "t"] +[28.534325, "o", "t"] +[28.584472, "o", "p"] +[28.641552, "o", "s"] +[28.691652, "o", ":"] +[28.741748, "o", "/"] +[28.791851, "o", "/"] +[28.841956, "o", "g"] +[28.892067, "o", "i"] +[28.947183, "o", "t"] +[29.050199, "o", "h"] +[29.100342, "o", "u"] +[29.223496, "o", "b"] +[29.273578, "o", "."] +[29.333666, "o", "c"] +[29.454774, "o", "o"] +[29.531893, "o", "m"] +[29.581901, "o", "/"] +[29.632077, "o", "e"] +[29.682187, "o", "d"] +[29.732276, "o", "g"] +[29.782377, "o", "e"] +[29.855454, "o", "l"] +[29.909573, "o", "e"] +[29.977692, "o", "s"] +[30.027803, "o", "s"] +[30.077895, "o", "s"] +[30.128002, "o", "y"] +[30.231162, "o", "s"] +[30.281229, "o", "/"] +[30.359303, "o", "c"] +[30.409383, "o", "o"] +[30.459506, "o", "n"] +[30.509603, "o", "s"] +[30.64375, "o", "t"] +[30.693798, "o", "e"] +[30.74391, "o", "l"] +[30.826028, "o", "l"] +[31.04817, "o", "a"] +[31.100227, "o", "t"] +[31.150363, "o", "i"] +[31.200473, "o", "o"] +[31.285574, "o", "n"] +[31.335675, "o", "/"] +[31.393786, "o", "r"] +[31.443861, "o", "e"] +[31.493993, "o", "l"] +[31.544086, "o", "e"] +[31.672217, "o", "a"] +[31.722318, "o", "s"] +[31.776413, "o", "e"] +[31.916535, "o", "s"] +[31.966612, "o", "/"] +[32.016742, "o", "l"] +[32.066905, "o", "a"] +[32.117018, "o", "t"] +[32.1721, "o", "e"] +[32.253216, "o", "s"] +[32.303282, "o", "t"] +[32.353422, "o", "/"] +[32.403516, "o", "d"] +[32.453599, "o", "o"] +[32.531729, "o", "w"] +[32.58178, "o", "n"] +[32.631909, "o", "l"] +[32.696025, "o", "o"] +[32.746104, "o", "a"] +[32.799206, "o", "d"] +[32.849295, "o", "/"] +[32.899406, "o", "c"] +[32.94949, "o", "o"] +[33.025626, "o", "n"] +[33.082733, "o", "s"] +[33.196865, "o", "t"] +[33.349977, "o", "e"] +[33.400083, "o", "l"] +[33.45019, "o", "l"] +[33.500303, "o", "a"] +[33.595398, "o", "t"] +[33.702507, "o", "i"] +[33.752616, "o", "o"] +[33.802708, "o", "n"] +[33.852808, "o", "."] +[33.951986, "o", "i"] +[34.002024, "o", "n"] +[34.052131, "o", "t"] +[34.102271, "o", "o"] +[34.152344, "o", "t"] +[34.202443, "o", "o"] +[34.252573, "o", "."] +[34.302659, "o", "j"] +[34.352774, "o", "s"] +[34.402878, "o", "o"] +[34.45299, "o", "n"] +[34.521051, "o", "l\r\n\u001b[?2004l\r"] +[34.999674, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[34.999921, "o", "#"] +[35.07111, "o", " "] +[35.121224, "o", "S"] +[35.30639, "o", "t"] +[35.356464, "o", "e"] +[35.406582, "o", "p"] +[35.456686, "o", " "] +[35.550776, "o", "3"] +[35.600866, "o", ":"] +[35.664, "o", " "] +[35.714099, "o", "C"] +[35.842202, "o", "h"] +[35.963349, "o", "e"] +[36.013392, "o", "c"] +[36.087525, "o", "k"] +[36.137616, "o", " "] +[36.223759, "o", "i"] +[36.311862, "o", "n"] +[36.361979, "o", "t"] +[36.412056, "o", "e"] +[36.462179, "o", "g"] +[36.545292, "o", "r"] +[36.595367, "o", "i"] +[36.645489, "o", "t"] +[36.727592, "o", "y"] +[36.777696, "o", " "] +[36.840784, "o", "o"] +[36.890915, "o", "f"] +[36.941006, "o", " "] +[37.011094, "o", "S"] +[37.061218, "o", "B"] +[37.111305, "o", "O"] +[37.161442, "o", "M\r\n\u001b[?2004l\r"] +[37.161549, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[38.16197, "o", "s"] +[38.212065, "o", "l"] +[38.262153, "o", "s"] +[38.312241, "o", "a"] +[38.36235, "o", "-"] +[38.533512, "o", "v"] +[38.583586, "o", "e"] +[38.63967, "o", "r"] +[38.689767, "o", "i"] +[38.739884, "o", "f"] +[38.790009, "o", "i"] +[38.840082, "o", "e"] +[38.917208, "o", "r"] +[38.96731, "o", " "] +[39.033401, "o", "v"] +[39.083493, "o", "e"] +[39.133626, "o", "r"] +[39.183739, "o", "i"] +[39.316862, "o", "f"] +[39.36696, "o", "y"] +[39.417005, "o", "-"] +[39.467151, "o", "a"] +[39.598223, "o", "r"] +[39.712365, "o", "t"] +[39.765432, "o", "i"] +[39.850551, "o", "f"] +[39.917674, "o", "a"] +[39.967765, "o", "c"] +[40.017871, "o", "t"] +[40.067973, "o", " "] +[40.118088, "o", "c"] +[40.231202, "o", "o"] +[40.311328, "o", "n"] +[40.361424, "o", "s"] +[40.411486, "o", "t"] +[40.481621, "o", "e"] +[40.531701, "o", "l"] +[40.581823, "o", "l"] +[40.631854, "o", "a"] +[40.681997, "o", "t"] +[40.732109, "o", "i"] +[40.782152, "o", "o"] +[40.832292, "o", "n"] +[40.882385, "o", "."] +[40.932444, "o", "s"] +[40.982563, "o", "p"] +[41.043694, "o", "d"] +[41.122782, "o", "x"] +[41.172873, "o", "."] +[41.22296, "o", "s"] +[41.280085, "o", "b"] +[41.330194, "o", "o"] +[41.380275, "o", "m"] +[41.430391, "o", " "] +[41.480492, "o", "-"] +[41.555589, "o", "-"] +[41.60566, "o", "p"] +[41.707785, "o", "r"] +[41.772886, "o", "o"] +[41.822955, "o", "v"] +[41.873085, "o", "e"] +[42.045138, "o", "n"] +[42.095282, "o", "a"] +[42.145394, "o", "n"] +[42.195495, "o", "c"] +[42.245563, "o", "e"] +[42.295691, "o", "-"] +[42.345777, "o", "p"] +[42.43086, "o", "a"] +[42.481006, "o", "t"] +[42.565107, "o", "h"] +[42.615202, "o", " "] +[42.665313, "o", "c"] +[42.715427, "o", "o"] +[42.765516, "o", "n"] +[42.820598, "o", "s"] +[42.870703, "o", "t"] +[43.003889, "o", "e"] +[43.053936, "o", "l"] +[43.104067, "o", "l"] +[43.15416, "o", "a"] +[43.20426, "o", "t"] +[43.315365, "o", "i"] +[43.365457, "o", "o"] +[43.415555, "o", "n"] +[43.465711, "o", "."] +[43.515792, "o", "i"] +[43.565941, "o", "n"] +[43.619986, "o", "t"] +[43.687106, "o", "o"] +[43.818197, "o", "t"] +[43.868326, "o", "o"] +[43.918419, "o", "."] +[43.968547, "o", "j"] +[44.018653, "o", "s"] +[44.099767, "o", "o"] +[44.149868, "o", "n"] +[44.199982, "o", "l"] +[44.250081, "o", " "] +[44.3642, "o", "-"] +[44.414207, "o", "-"] +[44.464374, "o", "s"] +[44.514497, "o", "o"] +[44.564611, "o", "u"] +[44.614674, "o", "r"] +[44.664811, "o", "c"] +[44.766905, "o", "e"] +[44.81698, "o", "-"] +[44.961093, "o", "u"] +[45.011194, "o", "r"] +[45.084322, "o", "i"] +[45.134363, "o", " "] +[45.184526, "o", "g"] +[45.234609, "o", "i"] +[45.284745, "o", "t"] +[45.334806, "o", "h"] +[45.384882, "o", "u"] +[45.435013, "o", "b"] +[45.485144, "o", "."] +[45.596248, "o", "c"] +[45.678352, "o", "o"] +[45.728428, "o", "m"] +[45.778523, "o", "/"] +[45.828654, "o", "e"] +[45.878738, "o", "d"] +[45.928873, "o", "g"] +[46.025961, "o", "e"] +[46.076052, "o", "l"] +[46.126164, "o", "e"] +[46.176273, "o", "s"] +[46.226335, "o", "s"] +[46.276508, "o", "s"] +[46.37963, "o", "y"] +[46.42971, "o", "s"] +[46.479802, "o", "/"] +[46.560926, "o", "c"] +[46.610965, "o", "o"] +[46.661106, "o", "n"] +[46.711196, "o", "s"] +[46.761322, "o", "t"] +[46.865416, "o", "e"] +[46.923504, "o", "l"] +[46.973615, "o", "l"] +[47.089723, "o", "a"] +[47.139819, "o", "t"] +[47.189891, "o", "i"] +[47.239992, "o", "o"] +[47.290139, "o", "n\r\n\u001b[?2004l\r"] +[47.982375, "o", "Verified signature against tlog entry index 57885093 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77a78f50198d0b0a4554049eec73d59a7e5e1f895c4ad5b3d202c62132aea18c4c1\r\n"] +[47.997219, "o", "Verified build using builder \"https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.9.0\" at commit 1d05f438ffa564d2e4ac3b071dbaf322e6d954ab\r\n"] +[47.997275, "o", "Verifying artifact constellation.spdx.sbom: PASSED\r\n\r\nPASSED: Verified SLSA provenance\r\n"] +[47.998419, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[47.998603, "o", "#"] +[48.088785, "o", " "] +[48.138885, "o", "S"] +[48.217918, "o", "t"] +[48.268062, "o", "e"] +[48.373203, "o", "p"] +[48.423265, "o", " "] +[48.47337, "o", "4"] +[48.523446, "o", ":"] +[48.584558, "o", " "] +[48.680705, "o", "I"] +[48.841802, "o", "n"] +[48.891906, "o", "s"] +[49.01802, "o", "t"] +[49.068149, "o", "a"] +[49.118278, "o", "l"] +[49.168333, "o", "l"] +[49.218431, "o", " "] +[49.268515, "o", "g"] +[49.31859, "o", "r"] +[49.368692, "o", "y"] +[49.49377, "o", "p"] +[49.543849, "o", "e"] +[49.593916, "o", " "] +[49.644038, "o", "("] +[49.694077, "o", "s"] +[49.744164, "o", "e"] +[49.794225, "o", "c"] +[49.844337, "o", "u"] +[49.92449, "o", "r"] +[49.974575, "o", "i"] +[50.024666, "o", "t"] +[50.074775, "o", "y"] +[50.12483, "o", " "] +[50.316965, "o", "s"] +[50.391113, "o", "c"] +[50.441153, "o", "a"] +[50.529288, "o", "n"] +[50.579374, "o", "n"] +[50.681516, "o", "e"] +[50.731593, "o", "r"] +[50.781733, "o", ")\r\n\u001b[?2004l\r"] +[50.781863, "o", "\u001b[?2004h"] +[50.781919, "o", "\u001b[38;2;139;4;221m$\u001b[0m "] +[51.782348, "o", "c"] +[51.881391, "o", "u"] +[51.955455, "o", "r"] +[52.018593, "o", "l"] +[52.068698, "o", " "] +[52.118772, "o", "-"] +[52.184813, "o", "s"] +[52.234919, "o", "L"] +[52.351046, "o", "O"] +[52.401144, "o", " "] +[52.600202, "o", "h"] +[52.650277, "o", "t"] +[52.700358, "o", "t"] +[52.750397, "o", "p"] +[52.800498, "o", "s"] +[52.850627, "o", ":"] +[52.946683, "o", "/"] +[52.99673, "o", "/"] +[53.046779, "o", "g"] +[53.096882, "o", "i"] +[53.147016, "o", "t"] +[53.197052, "o", "h"] +[53.24714, "o", "u"] +[53.297253, "o", "b"] +[53.347321, "o", "."] +[53.403409, "o", "c"] +[53.490509, "o", "o"] +[53.54056, "o", "m"] +[53.590619, "o", "/"] +[53.640708, "o", "a"] +[53.75379, "o", "n"] +[53.881871, "o", "c"] +[54.181998, "o", "h"] +[54.232015, "o", "o"] +[54.28308, "o", "r"] +[54.333173, "o", "e"] +[54.383301, "o", "/"] +[54.433389, "o", "g"] +[54.545535, "o", "r"] +[54.600652, "o", "y"] +[54.650734, "o", "p"] +[54.701875, "o", "e"] +[54.752016, "o", "/"] +[54.802149, "o", "r"] +[54.85225, "o", "e"] +[54.942348, "o", "l"] +[55.070445, "o", "e"] +[55.120565, "o", "a"] +[55.170705, "o", "s"] +[55.220803, "o", "e"] +[55.319939, "o", "s"] +[55.369999, "o", "/"] +[55.420126, "o", "d"] +[55.47017, "o", "o"] +[55.558311, "o", "w"] +[55.608407, "o", "n"] +[55.671523, "o", "l"] +[55.724627, "o", "o"] +[55.774695, "o", "a"] +[55.82481, "o", "d"] +[55.874899, "o", "/"] +[55.925068, "o", "v"] +[56.088184, "o", "0"] +[56.138275, "o", "."] +[56.198386, "o", "5"] +[56.248474, "o", "6"] +[56.298599, "o", "."] +[56.357724, "o", "0"] +[56.407848, "o", "/"] +[56.457923, "o", "g"] +[56.508035, "o", "r"] +[56.573121, "o", "y"] +[56.632192, "o", "p"] +[56.733328, "o", "e"] +[56.78341, "o", "_"] +[56.833519, "o", "0"] +[56.88364, "o", "."] +[56.933719, "o", "5"] +[57.03282, "o", "6"] +[57.082922, "o", "."] +[57.174059, "o", "0"] +[57.224126, "o", "_"] +[57.418282, "o", "l"] +[57.468348, "o", "i"] +[57.540451, "o", "n"] +[57.623562, "o", "u"] +[57.742662, "o", "x"] +[57.792774, "o", "_"] +[57.842853, "o", "a"] +[57.892949, "o", "m"] +[57.943065, "o", "d"] +[58.022208, "o", "6"] +[58.094259, "o", "4"] +[58.144368, "o", "."] +[58.194447, "o", "t"] +[58.244557, "o", "a"] +[58.294605, "o", "r"] +[58.344782, "o", "."] +[58.404852, "o", "g"] +[58.462042, "o", "z\r\n\u001b[?2004l\r"] +[59.482722, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[59.48292, "o", "t"] +[59.573075, "o", "a"] +[59.623201, "o", "r"] +[59.673286, "o", " "] +[59.733388, "o", "-"] +[59.783498, "o", "x"] +[59.833591, "o", "v"] +[59.88373, "o", "z"] +[59.933798, "o", "f"] +[59.983907, "o", " "] +[60.033995, "o", "g"] +[60.248148, "o", "r"] +[60.298228, "o", "y"] +[60.348332, "o", "p"] +[60.398418, "o", "e"] +[60.448526, "o", "_"] +[60.498587, "o", "0"] +[60.548763, "o", "."] +[60.599833, "o", "5"] +[60.649973, "o", "6"] +[60.700099, "o", "."] +[60.836203, "o", "0"] +[60.886285, "o", "_"] +[60.936398, "o", "l"] +[61.093519, "o", "i"] +[61.143601, "o", "n"] +[61.231707, "o", "u"] +[61.281818, "o", "x"] +[61.331909, "o", "_"] +[61.382013, "o", "a"] +[61.432112, "o", "m"] +[61.540237, "o", "d"] +[61.590317, "o", "6"] +[61.640442, "o", "4"] +[61.690531, "o", "."] +[61.782631, "o", "t"] +[61.832735, "o", "a"] +[61.993902, "o", "r"] +[62.044033, "o", "."] +[62.11714, "o", "g"] +[62.216257, "o", "z\r\n\u001b[?2004l\r"] +[62.218486, "o", "CHANGELOG.md\r\n"] +[62.218573, "o", "LICENSE\r\n"] +[62.218636, "o", "README.md\r\n"] +[62.218898, "o", "grype\r\n"] +[62.583512, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[62.583761, "o", "s"] +[62.633879, "o", "u"] +[62.684024, "o", "d"] +[62.786132, "o", "o"] +[62.836247, "o", " "] +[62.886333, "o", "i"] +[63.086489, "o", "n"] +[63.136564, "o", "s"] +[63.227678, "o", "t"] +[63.277776, "o", "a"] +[63.327871, "o", "l"] +[63.377976, "o", "l"] +[63.428082, "o", " "] +[63.523207, "o", "g"] +[63.573292, "o", "r"] +[63.623386, "o", "y"] +[63.680486, "o", "p"] +[63.730568, "o", "e"] +[63.780681, "o", " "] +[63.992838, "o", "/"] +[64.059976, "o", "u"] +[64.122052, "o", "s"] +[64.17215, "o", "r"] +[64.222254, "o", "/"] +[64.272401, "o", "l"] +[64.441502, "o", "o"] +[64.496554, "o", "c"] +[64.546644, "o", "a"] +[64.616771, "o", "l"] +[64.666882, "o", "/"] +[64.73901, "o", "b"] +[64.789087, "o", "i"] +[64.855211, "o", "n"] +[64.905325, "o", "/"] +[65.076452, "o", "g"] +[65.126538, "o", "r"] +[65.176681, "o", "y"] +[65.325805, "o", "p"] +[65.375926, "o", "e\r\n\u001b[?2004l\r"] +[65.39875, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[66.399189, "o", "g"] +[66.44936, "o", "r"] +[66.499442, "o", "y"] +[66.549537, "o", "p"] +[66.599637, "o", "e"] +[66.649755, "o", " "] +[66.700867, "o", "-"] +[66.750972, "o", "-"] +[66.866108, "o", "h"] +[66.9482, "o", "e"] +[66.998294, "o", "l"] +[67.107435, "o", "p\r\n\u001b[?2004l\r"] +[67.133287, "o", "A vulnerability scanner for container images, filesystems, and SBOMs.\r\n\r\nSupports the following image sources:\r\n grype yourrepo/yourimage:tag defaults to using images from a Docker daemon\r\n grype path/to/yourproject a Docker tar, OCI tar, OCI directory, SIF container, or generic filesystem directory\r\n grype attestation.json --key cosign.pub extract and scan SBOM from attestation file\r\n\r\nYou can also explicitly specify the scheme to use:\r\n grype podman:yourrepo/yourimage:tag explicitly use the Podman daemon\r\n grype docker:yourrepo/yourimage:tag explicitly use the Docker daemon\r\n grype docker-archive:path/to/yourimage.tar use a tarball from disk for archives created from \"docker save\"\r\n grype oci-archive:path/to/yourimage.tar use a tarball from disk for OCI archives (from Podman or otherwise)\r\n grype oci-dir:path/to/yourimage read directly from a path on disk for OCI layout directories (from Skopeo or otherwise)\r\n grype"] +[67.13335, "o", " singularity:path/to/yourimage.sif read directly from a Singularity Image Format (SIF) container on disk\r\n grype dir:path/to/yourproject read directly from a path on disk (any directory)\r\n grype sbom:path/to/syft.json read Syft JSON from path on disk\r\n grype registry:yourrepo/yourimage:tag pull image directly from a registry (no container runtime required)\r\n grype att:attestation.json --key cosign.pub explicitly use the input as an attestation\r\n grype purl:path/to/purl/file read a newline separated file of purls from a path on disk\r\n\r\nYou can also pipe in Syft JSON directly:\r\n\tsyft yourimage:tag -o json | grype\r\n\r\n"] +[67.134367, "o", "Usage:\r\n grype [IMAGE] [flags]\r\n grype [command]\r\n\r\nAvailable Commands:\r\n completion Generate a shell completion for Grype (listing local docker images)\r\n db vulnerability database operations\r\n help Help about any command\r\n version show the version\r\n\r\nFlags:\r\n --add-cpes-if-none generate CPEs for packages with no CPE data\r\n --by-cve orient results by CVE instead of the original vulnerability ID when possible\r\n -c, --config string application config file\r\n --distro string distro to match against in the format: :\r\n --exclude stringArray exclude paths from being scanned using a glob expression\r\n -f, --fail-on string set the return code to 1 if a vulnerability is found with a severity >= the given severity, options=[negligible low medium high critical]\r\n --file string file to write the report output to (default is STDOUT)\r\n -h, --help help for grype\r\n --key string "] +[67.134512, "o", " File path to a public key to validate attestation\r\n --only-fixed ignore matches for vulnerabilities that are not fixed\r\n --only-notfixed ignore matches for vulnerabilities that are fixed\r\n -o, --output string report output formatter, formats=[json table cyclonedx cyclonedx-json sarif template], deprecated formats=[embedded-cyclonedx-vex-json embedded-cyclonedx-vex-xml]\r\n --platform string an optional platform specifier for container image sources (e.g. 'linux/arm64', 'linux/arm64/v8', 'arm64', 'linux')\r\n -q, --quiet suppress all logging output\r\n -s, --scope string selection of layers to analyze, options=[Squashed AllLayers] (default \"Squashed\")\r\n --show-suppressed show suppressed/ignored vulnerabilities in the output (only supported with table output format)\r\n -t, --template string specify the path to a Go template file (requires 'template' output to be selected)\r\n -v, --verbose count increase verbo"] +[67.13459, "o", "sity (-v = info, -vv = debug)\r\n\r\nUse \"grype [command] --help\" for more information about a command.\r\n"] +[67.135666, "o", "\u001b[?2004h"] +[67.135682, "o", "\u001b[38;2;139;4;221m$\u001b[0m "] +[68.136095, "o", "#"] +[68.238218, "o", " "] +[68.288378, "o", "S"] +[68.338457, "o", "t"] +[68.388622, "o", "e"] +[68.576751, "o", "p"] +[68.62676, "o", " "] +[68.771975, "o", "5"] +[68.822001, "o", ":"] +[68.872118, "o", " "] +[68.934225, "o", "C"] +[68.984343, "o", "h"] +[69.052468, "o", "e"] +[69.119589, "o", "c"] +[69.169675, "o", "k"] +[69.219783, "o", " "] +[69.284903, "o", "f"] +[69.33499, "o", "o"] +[69.607177, "o", "r"] +[69.657213, "o", " "] +[69.707329, "o", "v"] +[69.757433, "o", "u"] +[69.89558, "o", "l"] +[69.945665, "o", "n"] +[69.995771, "o", "e"] +[70.045863, "o", "r"] +[70.197003, "o", "a"] +[70.247062, "o", "b"] +[70.378228, "o", "i"] +[70.428335, "o", "l"] +[70.478431, "o", "i"] +[70.528523, "o", "t"] +[70.597668, "o", "i"] +[70.720806, "o", "e"] +[70.770913, "o", "s\r\n\u001b[?2004l\r"] +[70.771054, "o", "\u001b[?2004h"] +[70.771103, "o", "\u001b[38;2;139;4;221m$\u001b[0m "] +[71.771528, "o", "g"] +[71.882659, "o", "r"] +[71.932765, "o", "y"] +[71.999894, "o", "p"] +[72.049983, "o", "e"] +[72.100078, "o", " "] +[72.150204, "o", "c"] +[72.405363, "o", "o"] +[72.538448, "o", "n"] +[72.702565, "o", "s"] +[72.752703, "o", "t"] +[72.8378, "o", "e"] +[72.896879, "o", "l"] +[72.946963, "o", "l"] +[72.997074, "o", "a"] +[73.047183, "o", "t"] +[73.097283, "o", "i"] +[73.242451, "o", "o"] +[73.405569, "o", "n"] +[73.455655, "o", "."] +[73.505736, "o", "s"] +[73.555888, "o", "p"] +[73.605983, "o", "d"] +[73.679102, "o", "x"] +[73.729262, "o", "."] +[73.790379, "o", "s"] +[73.846481, "o", "b"] +[73.896578, "o", "o"] +[73.946706, "o", "m"] +[73.996848, "o", " "] +[74.046866, "o", "-"] +[74.096937, "o", "o"] +[74.14701, "o", " "] +[74.197095, "o", "t"] +[74.247171, "o", "a"] +[74.299252, "o", "b"] +[74.370336, "o", "l"] +[74.601468, "o", "e"] +[74.651494, "o", " "] +[74.709569, "o", "-"] +[74.773696, "o", "q\r\n\u001b[?2004l\r"] +[84.703424, "o", "No vulnerabilities found\r\n"] +[84.706167, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] diff --git a/docs/static/assets/configure-cluster.cast b/docs/static/assets/configure-cluster.cast index 3f8983d97..9e1144436 100644 --- a/docs/static/assets/configure-cluster.cast +++ b/docs/static/assets/configure-cluster.cast @@ -1,288 +1,287 @@ -{"version": 2, "width": 126, "height": 61, "timestamp": 1700561372, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} -[0.008412, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[0.008896, "o", "#"] -[0.141011, "o", " "] -[0.191056, "o", "S"] -[0.241065, "o", "t"] -[0.312122, "o", "e"] -[0.362296, "o", "p"] -[0.412401, "o", " "] -[0.532497, "o", "1"] -[0.582865, "o", ":"] -[0.632826, "o", " "] -[0.683072, "o", "C"] -[0.789998, "o", "r"] -[0.839997, "o", "e"] -[0.922076, "o", "a"] -[0.972114, "o", "t"] -[1.02226, "o", "e"] -[1.07243, "o", " "] -[1.147356, "o", "a"] -[1.197453, "o", " "] -[1.247846, "o", "c"] -[1.371939, "o", "o"] -[1.421936, "o", "n"] -[1.48295, "o", "f"] -[1.589121, "o", "i"] -[1.639096, "o", "g"] -[1.689272, "o", "u"] -[1.739326, "o", "r"] -[1.789372, "o", "a"] -[1.83983, "o", "t"] -[1.889875, "o", "i"] -[1.990801, "o", "o"] -[2.04088, "o", "n"] -[2.090835, "o", " "] -[2.141089, "o", "f"] -[2.191295, "o", "i"] -[2.241321, "o", "l"] -[2.291323, "o", "e"] -[2.341395, "o", " "] -[2.398484, "o", "f"] -[2.463524, "o", "o"] -[2.513615, "o", "r"] -[2.56371, "o", " "] -[2.613785, "o", "C"] -[2.70086, "o", "o"] -[2.751007, "o", "n"] -[2.80111, "o", "s"] -[2.859191, "o", "t"] -[2.90928, "o", "e"] -[3.01535, "o", "l"] -[3.065394, "o", "l"] -[3.115544, "o", "a"] -[3.195882, "o", "t"] -[3.296717, "o", "i"] -[3.346807, "o", "o"] -[3.40301, "o", "n"] -[3.403105, "o", "\r\n\u001b[?2004l\r"] -[3.403157, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[4.403509, "o", "c"] -[4.485672, "o", "o"] -[4.659742, "o", "n"] -[4.709962, "o", "s"] -[4.826943, "o", "t"] -[4.876961, "o", "e"] -[5.000164, "o", "l"] -[5.050233, "o", "l"] -[5.291386, "o", "a"] -[5.397519, "o", "t"] -[5.455544, "o", "i"] -[5.52791, "o", "o"] -[5.577797, "o", "n"] -[5.627896, "o", " "] -[5.678015, "o", "c"] -[5.761106, "o", "o"] -[5.811202, "o", "n"] -[5.969293, "o", "f"] -[6.019493, "o", "i"] -[6.069627, "o", "g"] -[6.119746, "o", " "] -[6.169832, "o", "g"] -[6.225045, "o", "e"] -[6.274956, "o", "n"] -[6.325084, "o", "e"] -[6.485173, "o", "r"] -[6.535254, "o", "a"] -[6.722457, "o", "t"] -[6.809537, "o", "e"] -[6.859733, "o", " "] -[6.909784, "o", "g"] -[6.959906, "o", "c"] -[7.010007, "o", "p\r\n\u001b[?2004l\r"] -[7.166504, "o", "Config file written to constellation-conf.yaml\r\nPlease fill in your CSP-specific configuration before proceeding.\r\n"] -[7.166941, "o", "State file written to constellation-state.yaml\r\nFor more information refer to the documentation:\r\n\thttps://docs.edgeless.systems/constellation/getting-started/first-steps\r\n"] -[7.16943, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[7.169656, "o", "#"] -[7.219987, "o", " "] -[7.269995, "o", "S"] -[7.391392, "o", "t"] -[7.441178, "o", "e"] -[7.491323, "o", "p"] -[7.54152, "o", " "] -[7.591425, "o", "2"] -[7.641572, "o", ":"] -[7.691721, "o", " "] -[7.741771, "o", "C"] -[7.792011, "o", "r"] -[7.844995, "o", "e"] -[7.92005, "o", "a"] -[7.972202, "o", "t"] -[8.022319, "o", "e"] -[8.072366, "o", " "] -[8.12278, "o", "y"] -[8.175691, "o", "o"] -[8.225779, "o", "u"] -[8.27612, "o", "r"] -[8.326092, "o", " "] -[8.376368, "o", "c"] -[8.456517, "o", "l"] -[8.506473, "o", "u"] -[8.667582, "o", "s"] -[8.717845, "o", "t"] -[8.767746, "o", "e"] -[8.817983, "o", "r"] -[8.868071, "o", "'"] -[8.918192, "o", "s"] -[8.968184, "o", " "] -[9.018438, "o", "I"] -[9.068576, "o", "A"] -[9.118755, "o", "M"] -[9.168833, "o", " "] -[9.230972, "o", "c"] -[9.281053, "o", "o"] -[9.331154, "o", "n"] -[9.381345, "o", "f"] -[9.43144, "o", "i"] -[9.481588, "o", "g"] -[9.531532, "o", "u"] -[9.605793, "o", "r"] -[9.655912, "o", "a"] -[9.737118, "o", "t"] -[9.78698, "o", "i"] -[9.871392, "o", "o"] -[9.921397, "o", "n\r\n\u001b[?2004l\r"] -[9.921619, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[10.921999, "o", "c"] -[10.972233, "o", "o"] -[11.022296, "o", "n"] -[11.072463, "o", "s"] -[11.162663, "o", "t"] -[11.253503, "o", "e"] -[11.304004, "o", "l"] -[11.353919, "o", "l"] -[11.404132, "o", "a"] -[11.508049, "o", "t"] -[11.657242, "o", "i"] -[11.707306, "o", "o"] -[11.75752, "o", "n"] -[11.807701, "o", " "] -[11.857807, "o", "i"] -[11.907985, "o", "a"] -[11.973078, "o", "m"] -[12.023268, "o", " "] -[12.110336, "o", "c"] -[12.160538, "o", "r"] -[12.215558, "o", "e"] -[12.265739, "o", "a"] -[12.460861, "o", "t"] -[12.632867, "o", "e"] -[12.683139, "o", " "] -[12.73317, "o", "g"] -[12.816267, "o", "c"] -[12.866316, "o", "p"] -[12.916677, "o", " "] -[12.966705, "o", "-"] -[13.016861, "o", "-"] -[13.067042, "o", "u"] -[13.117001, "o", "p"] -[13.167105, "o", "d"] -[13.217276, "o", "a"] -[13.267344, "o", "t"] -[13.317575, "o", "e"] -[13.367721, "o", "-"] -[13.41784, "o", "c"] -[13.467969, "o", "o"] -[13.539005, "o", "n"] -[13.58906, "o", "f"] -[13.639169, "o", "i"] -[13.71734, "o", "g"] -[13.767472, "o", " "] -[13.829508, "o", "-"] -[13.879794, "o", "-"] -[13.929856, "o", "p"] -[13.979828, "o", "r"] -[14.139912, "o", "o"] -[14.213897, "o", "j"] -[14.278289, "o", "e"] -[14.328253, "o", "c"] -[14.378493, "o", "t"] -[14.436824, "o", "I"] -[14.486798, "o", "D"] -[14.536988, "o", " "] -[14.586946, "o", "c"] -[14.637167, "o", "o"] -[14.70242, "o", "n"] -[14.817468, "o", "s"] -[14.969608, "o", "t"] -[15.26961, "o", "e"] -[15.394779, "o", "l"] -[15.444766, "o", "l"] -[15.494817, "o", "a"] -[15.544943, "o", "t"] -[15.595131, "o", "i"] -[15.645148, "o", "o"] -[15.695185, "o", "n"] -[15.745307, "o", "-"] -[15.795362, "o", "3"] -[15.845531, "o", "3"] -[15.895558, "o", "1"] -[15.980129, "o", "6"] -[16.062066, "o", "1"] -[16.166928, "o", "3"] -[16.217042, "o", " "] -[16.280407, "o", "-"] -[16.413607, "o", "-"] -[16.469513, "o", "s"] -[16.519692, "o", "e"] -[16.5729, "o", "r"] -[16.622937, "o", "v"] -[16.700143, "o", "i"] -[16.750139, "o", "c"] -[16.800392, "o", "e"] -[16.850593, "o", "A"] -[16.900458, "o", "c"] -[16.95076, "o", "c"] -[17.000784, "o", "o"] -[17.051057, "o", "u"] -[17.101025, "o", "n"] -[17.151049, "o", "t"] -[17.219185, "o", "I"] -[17.269365, "o", "D"] -[17.31958, "o", " "] -[17.369589, "o", "c"] -[17.419992, "o", "o"] -[17.487042, "o", "n"] -[17.537118, "o", "s"] -[17.587108, "o", "t"] -[17.658331, "o", "e"] -[17.816442, "o", "l"] -[17.8936, "o", "l"] -[17.943785, "o", "a"] -[17.993874, "o", "t"] -[18.04807, "o", "i"] -[18.098034, "o", "o"] -[18.14813, "o", "n"] -[18.198116, "o", "-"] -[18.368509, "o", "d"] -[18.41847, "o", "e"] -[18.468432, "o", "m"] -[18.518519, "o", "o"] -[18.56855, "o", " "] -[18.686928, "o", "-"] -[18.736922, "o", "-"] -[18.793085, "o", "z"] -[18.843216, "o", "o"] -[18.893286, "o", "n"] -[18.967495, "o", "e"] -[19.017573, "o", " "] -[19.083707, "o", "e \r"] -[19.134857, "o", "u"] -[19.237, "o", "r"] -[19.310026, "o", "o"] -[19.418943, "o", "p"] -[19.478061, "o", "e"] -[19.528191, "o", "-"] -[19.578336, "o", "w"] -[19.628339, "o", "e"] -[19.69946, "o", "s"] -[19.749537, "o", "t"] -[19.946923, "o", "3"] -[19.997079, "o", "-"] -[20.062985, "o", "b\r\n\u001b[?2004l\r"] -[20.093986, "o", "The following IAM configuration will be created:\r\n\r\nProject ID:\t\tconstellation-331613\r\nService Account ID:\tconstellation-demo\r\nRegion:\t\t\teurope-west3\r\nZone:\t\t\teurope-west3-b\r\n\r\nDo you want to create the configuration? [y/n]: "] -[20.094258, "o", "y\r\n"] -[20.094798, "o", "The configuration file \"constellation-conf.yaml\" will be automatically updated with the IAM values and zone/region information.\r\n"] -[56.319787, "o", "\r\n"] -[56.321054, "o", "Your IAM configuration was created and filled into constellation-conf.yaml successfully.\r\n"] -[56.323816, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +{"version": 2, "width": 199, "height": 20, "timestamp": 1703674736, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} +[0.004032, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[0.0047, "o", "#"] +[0.136425, "o", " "] +[0.18652, "o", "S"] +[0.236646, "o", "t"] +[0.307751, "o", "e"] +[0.357879, "o", "p"] +[0.407924, "o", " "] +[0.528109, "o", "1"] +[0.578222, "o", ":"] +[0.628331, "o", " "] +[0.678432, "o", "C"] +[0.785541, "o", "r"] +[0.835629, "o", "e"] +[0.917737, "o", "a"] +[0.967837, "o", "t"] +[1.017913, "o", "e"] +[1.068028, "o", " "] +[1.143165, "o", "a"] +[1.193321, "o", " "] +[1.24335, "o", "c"] +[1.367532, "o", "o"] +[1.417521, "o", "n"] +[1.478682, "o", "f"] +[1.584787, "o", "i"] +[1.634917, "o", "g"] +[1.68502, "o", "u"] +[1.735123, "o", "r"] +[1.785259, "o", "a"] +[1.835365, "o", "t"] +[1.885473, "o", "i"] +[1.986584, "o", "o"] +[2.036683, "o", "n"] +[2.086792, "o", " "] +[2.136903, "o", "f"] +[2.186955, "o", "i"] +[2.237061, "o", "l"] +[2.287153, "o", "e"] +[2.337246, "o", " "] +[2.394374, "o", "f"] +[2.459474, "o", "o"] +[2.509565, "o", "r"] +[2.559701, "o", " "] +[2.609766, "o", "C"] +[2.696897, "o", "o"] +[2.746999, "o", "n"] +[2.797107, "o", "s"] +[2.855202, "o", "t"] +[2.905286, "o", "e"] +[3.01141, "o", "l"] +[3.061575, "o", "l"] +[3.111629, "o", "a"] +[3.191802, "o", "t"] +[3.292869, "o", "i"] +[3.342912, "o", "o"] +[3.399061, "o", "n\r\n\u001b[?2004l\r"] +[3.399191, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[4.399632, "o", "c"] +[4.481744, "o", "o"] +[4.655888, "o", "n"] +[4.705977, "o", "s"] +[4.823081, "o", "t"] +[4.873147, "o", "e"] +[4.9963, "o", "l"] +[5.046371, "o", "l"] +[5.287489, "o", "a"] +[5.393609, "o", "t"] +[5.451736, "o", "i"] +[5.523853, "o", "o"] +[5.573935, "o", "n"] +[5.624052, "o", " "] +[5.674145, "o", "c"] +[5.757261, "o", "o"] +[5.807352, "o", "n"] +[5.965528, "o", "f"] +[6.015715, "o", "i"] +[6.065677, "o", "g"] +[6.115839, "o", " "] +[6.165908, "o", "g"] +[6.22102, "o", "e"] +[6.271112, "o", "n"] +[6.321249, "o", "e"] +[6.481384, "o", "r"] +[6.531481, "o", "a"] +[6.718632, "o", "t"] +[6.80562, "o", "e"] +[6.855783, "o", " "] +[6.905964, "o", "g"] +[6.956034, "o", "c"] +[7.006121, "o", "p\r\n\u001b[?2004l\r"] +[7.035317, "o", "Config file written to constellation-conf.yaml\r\nPlease fill in your CSP-specific configuration before proceeding.\r\n"] +[7.035572, "o", "State file written to constellation-state.yaml\r\nFor more information refer to the documentation:\r\n\thttps://docs.edgeless.systems/constellation/getting-started/first-steps\r\n"] +[7.038178, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[8.038633, "o", "#"] +[8.088712, "o", " "] +[8.138817, "o", "S"] +[8.259909, "o", "t"] +[8.31, "o", "e"] +[8.360132, "o", "p"] +[8.410205, "o", " "] +[8.460368, "o", "2"] +[8.510439, "o", ":"] +[8.560548, "o", " "] +[8.610654, "o", "C"] +[8.660765, "o", "r"] +[8.713833, "o", "e"] +[8.788945, "o", "a"] +[8.84102, "o", "t"] +[8.891088, "o", "e"] +[8.941238, "o", " "] +[8.991346, "o", "y"] +[9.044459, "o", "o"] +[9.094568, "o", "u"] +[9.144646, "o", "r"] +[9.194761, "o", " "] +[9.244875, "o", "c"] +[9.324972, "o", "l"] +[9.375092, "o", "u"] +[9.536214, "o", "s"] +[9.586296, "o", "t"] +[9.6364, "o", "e"] +[9.686521, "o", "r"] +[9.736603, "o", "'"] +[9.786759, "o", "s"] +[9.836852, "o", " "] +[9.886958, "o", "I"] +[9.937051, "o", "A"] +[9.987165, "o", "M"] +[10.037293, "o", " "] +[10.099391, "o", "c"] +[10.149491, "o", "o"] +[10.199613, "o", "n"] +[10.249678, "o", "f"] +[10.299784, "o", "i"] +[10.349852, "o", "g"] +[10.399952, "o", "u"] +[10.474072, "o", "r"] +[10.524166, "o", "a"] +[10.605278, "o", "t"] +[10.655383, "o", "i"] +[10.739508, "o", "o"] +[10.789624, "o", "n\r\n\u001b[?2004l\r"] +[10.789745, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[11.790173, "o", "c"] +[11.840254, "o", "o"] +[11.890331, "o", "n"] +[11.940431, "o", "s"] +[12.030563, "o", "t"] +[12.121623, "o", "e"] +[12.171772, "o", "l"] +[12.221873, "o", "l"] +[12.27199, "o", "a"] +[12.376097, "o", "t"] +[12.525194, "o", "i"] +[12.575311, "o", "o"] +[12.62537, "o", "n"] +[12.675493, "o", " "] +[12.72561, "o", "i"] +[12.775709, "o", "a"] +[12.840809, "o", "m"] +[12.890909, "o", " "] +[12.978004, "o", "c"] +[13.028105, "o", "r"] +[13.083183, "o", "e"] +[13.133317, "o", "a"] +[13.328475, "o", "t"] +[13.500571, "o", "e"] +[13.550589, "o", " "] +[13.600733, "o", "g"] +[13.683842, "o", "c"] +[13.733945, "o", "p"] +[13.784006, "o", " "] +[13.83407, "o", "-"] +[13.884212, "o", "-"] +[13.934298, "o", "u"] +[13.984399, "o", "p"] +[14.034493, "o", "d"] +[14.084615, "o", "a"] +[14.134706, "o", "t"] +[14.184824, "o", "e"] +[14.234934, "o", "-"] +[14.285033, "o", "c"] +[14.335095, "o", "o"] +[14.406208, "o", "n"] +[14.456318, "o", "f"] +[14.506333, "o", "i"] +[14.58452, "o", "g"] +[14.634566, "o", " "] +[14.696712, "o", "-"] +[14.746804, "o", "-"] +[14.796923, "o", "p"] +[14.846966, "o", "r"] +[15.007094, "o", "o"] +[15.081205, "o", "j"] +[15.145334, "o", "e"] +[15.195449, "o", "c"] +[15.245547, "o", "t"] +[15.303651, "o", "I"] +[15.353709, "o", "D"] +[15.403816, "o", " "] +[15.453927, "o", "c"] +[15.504006, "o", "o"] +[15.569101, "o", "n"] +[15.684325, "o", "s"] +[15.836396, "o", "t"] +[16.136569, "o", "e"] +[16.261635, "o", "l"] +[16.311702, "o", "l"] +[16.361785, "o", "a"] +[16.411903, "o", "t"] +[16.462006, "o", "i"] +[16.512109, "o", "o"] +[16.562191, "o", "n"] +[16.612327, "o", "-"] +[16.662437, "o", "3"] +[16.712526, "o", "3"] +[16.762669, "o", "1"] +[16.846792, "o", "6"] +[16.928893, "o", "1"] +[17.033982, "o", "3"] +[17.084126, "o", " "] +[17.147145, "o", "-"] +[17.280363, "o", "-"] +[17.336469, "o", "s"] +[17.386543, "o", "e"] +[17.439609, "o", "r"] +[17.489728, "o", "v"] +[17.566833, "o", "i"] +[17.61694, "o", "c"] +[17.667028, "o", "e"] +[17.717124, "o", "A"] +[17.767239, "o", "c"] +[17.817365, "o", "c"] +[17.867472, "o", "o"] +[17.917573, "o", "u"] +[17.967659, "o", "n"] +[18.017751, "o", "t"] +[18.085884, "o", "I"] +[18.135969, "o", "D"] +[18.18609, "o", " "] +[18.236186, "o", "c"] +[18.286291, "o", "o"] +[18.353396, "o", "n"] +[18.403499, "o", "s"] +[18.453618, "o", "t"] +[18.524736, "o", "e"] +[18.682906, "o", "l"] +[18.759976, "o", "l"] +[18.810088, "o", "a"] +[18.860234, "o", "t"] +[18.91433, "o", "i"] +[18.964426, "o", "o"] +[19.014564, "o", "n"] +[19.064793, "o", "-"] +[19.23487, "o", "d"] +[19.284893, "o", "e"] +[19.335003, "o", "m"] +[19.385139, "o", "o"] +[19.435222, "o", " "] +[19.553365, "o", "-"] +[19.603414, "o", "-"] +[19.659537, "o", "z"] +[19.709637, "o", "o"] +[19.759731, "o", "n"] +[19.833864, "o", "e"] +[19.883996, "o", " "] +[19.950136, "o", "e"] +[20.001271, "o", "u"] +[20.103368, "o", "r"] +[20.176485, "o", "o"] +[20.285563, "o", "p"] +[20.344673, "o", "e"] +[20.394801, "o", "-"] +[20.444906, "o", "w"] +[20.495018, "o", "e"] +[20.566087, "o", "s"] +[20.616229, "o", "t"] +[20.813354, "o", "3"] +[20.863452, "o", "-"] +[20.929523, "o", "b\r\n\u001b[?2004l\r"] +[20.959012, "o", "The following IAM configuration will be created:\r\n\r\nProject ID:\t\tconstellation-331613\r\nService Account ID:\tconstellation-demo\r\nRegion:\t\t\teurope-west3\r\nZone:\t\t\teurope-west3-b\r\n\r\nDo you want to create the configuration? [y/n]: "] +[21.959455, "o", "y\r\n"] +[21.960106, "o", "The configuration file \"constellation-conf.yaml\" will be automatically updated with the IAM values and zone/region information.\r\n"] +[53.669359, "o", "\r\n"] +[53.670722, "o", "Your IAM configuration was created and filled into constellation-conf.yaml successfully.\r\n"] +[53.672522, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] diff --git a/docs/static/assets/create-cluster.cast b/docs/static/assets/create-cluster.cast index e64723eba..3c9f00a95 100644 --- a/docs/static/assets/create-cluster.cast +++ b/docs/static/assets/create-cluster.cast @@ -1,291 +1,240 @@ -{"version": 2, "width": 126, "height": 61, "timestamp": 1700561429, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} -[0.007081, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[0.00763, "o", "#"] -[0.139692, "o", " "] -[0.189803, "o", "S"] -[0.239954, "o", "t"] -[0.311167, "o", "e"] -[0.36132, "o", "p"] -[0.411522, "o", " "] -[0.531647, "o", "1"] -[0.581499, "o", ":"] -[0.631928, "o", " "] -[0.681658, "o", "C"] -[0.788871, "o", "r"] -[0.839041, "o", "e"] -[0.921404, "o", "a"] -[0.971242, "o", "t"] -[1.021585, "o", "e"] -[1.071831, "o", " "] -[1.146806, "o", "c"] -[1.236105, "o", "l"] -[1.286094, "o", "o"] -[1.41038, "o", "u"] -[1.460239, "o", "d"] -[1.510473, "o", " "] -[1.616598, "o", "e"] -[1.666746, "o", "n"] -[1.716826, "o", "v"] -[1.766986, "o", "i"] -[1.817138, "o", "r"] -[1.867269, "o", "o"] -[1.917535, "o", "n"] -[2.018727, "o", "m"] -[2.068618, "o", "e"] -[2.118596, "o", "n"] -[2.168694, "o", "t\r\n\u001b[?2004l\r"] -[2.168809, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[3.169217, "o", "c"] -[3.219424, "o", "o"] -[3.269516, "o", "n"] -[3.405842, "o", "s"] -[3.462794, "o", "t"] -[3.528028, "o", "e"] -[3.57813, "o", "l"] -[3.644284, "o", "l"] -[3.694407, "o", "a"] -[3.781681, "o", "t"] -[3.831599, "o", "i"] -[3.881896, "o", "o"] -[3.939879, "o", "n"] -[3.989936, "o", " "] -[4.096111, "o", "c"] -[4.146206, "o", "r"] -[4.196614, "o", "e"] -[4.276676, "o", "a"] -[4.377628, "o", "t"] -[4.42779, "o", "e\r\n\u001b[?2004l\r"] -[6.219914, "o", "The following Constellation cluster will be created:\r\n 3 control-plane nodes of type n2d-standard-4 will be created.\r\n 1 worker node of type n2d-standard-4 will be created.\r\nDo you want to create this cluster? [y/n]: "] -[6.220165, "o", "y\r\n"] -[178.19682, "o", "Your Constellation cluster was created successfully.\r\n"] -[178.204846, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[178.204962, "o", "#"] -[178.323348, "o", " "] -[178.405545, "o", "S"] -[178.579491, "o", "t"] -[178.629584, "o", "e"] -[178.746568, "o", "p"] -[178.796698, "o", " "] -[178.919763, "o", "2"] -[178.969874, "o", ":"] -[179.210997, "o", " "] -[179.3172, "o", "I"] -[179.375465, "o", "n"] -[179.447486, "o", "i"] -[179.497542, "o", "t"] -[179.547676, "o", "i"] -[179.59776, "o", "a"] -[179.680752, "o", "l"] -[179.730714, "o", "i"] -[179.889056, "o", "z"] -[179.939156, "o", "e"] -[179.989393, "o", " "] -[180.039493, "o", "C"] -[180.089424, "o", "o"] -[180.144594, "o", "n"] -[180.194704, "o", "s"] -[180.244674, "o", "t"] -[180.404814, "o", "e"] -[180.455047, "o", "l"] -[180.642165, "o", "l"] -[180.729192, "o", "a"] -[180.779398, "o", "t"] -[180.829704, "o", "i"] -[180.879747, "o", "o"] -[180.929822, "o", "n\r\n\u001b[?2004l\r"] -[180.930033, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[181.930623, "o", "c"] -[181.980808, "o", "o"] -[182.030803, "o", "n"] -[182.151862, "o", "s"] -[182.201848, "o", "t"] -[182.25196, "o", "e"] -[182.307304, "o", "l"] -[182.357241, "o", "l"] -[182.458537, "o", "a"] -[182.508481, "o", "t"] -[182.558867, "o", "i"] -[182.608707, "o", "o"] -[182.661713, "o", "n"] -[182.711841, "o", " "] -[182.763937, "o", "a"] -[182.814323, "o", "p"] -[182.881238, "o", "p"] -[182.931444, "o", "l"] -[182.984462, "o", "y\r\n"] -[182.984545, "o", "\u001b[?2004l\r"] -[183.019888, "o", "Using community license.\r\n"] -[185.395102, "o", "For details, see https://docs.edgeless.systems/constellation/overview/license\r\n"] -[193.581442, "o", "Your Constellation master secret was successfully written to \"constellation-mastersecret.json\"\r\n"] -[643.029028, "o", "Your Constellation cluster was successfully initialized.\r\n\r\nConstellation cluster identifier da0f1ad905a9e492b816b0321672a811335aec924d38802ec0b3da3f5fa4c011\r\nKubernetes configuration constellation-admin.conf\r\n\r\nYou can now connect to your cluster by executing:\r\n\texport KUBECONFIG=\"/constellation/constellation-admin.conf\"\r\n\r\n"] -[643.036006, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[643.036129, "o", "#"] -[643.086612, "o", " "] -[643.155611, "o", "W"] -[643.205707, "o", "a"] -[643.28608, "o", "i"] -[643.336046, "o", "t"] -[643.386109, "o", " "] -[643.436272, "o", "f"] -[643.486492, "o", "o"] -[643.536477, "o", "r"] -[643.586587, "o", " "] -[643.636756, "o", "c"] -[643.71174, "o", "l"] -[643.762017, "o", "u"] -[643.81209, "o", "s"] -[643.862387, "o", "t"] -[643.91247, "o", "e"] -[643.974528, "o", "r"] -[644.024671, "o", " "] -[644.074667, "o", "t"] -[644.124594, "o", "o"] -[644.174737, "o", " "] -[644.224908, "o", "f"] -[644.275214, "o", "i"] -[644.34915, "o", "n"] -[644.399437, "o", "i"] -[644.480587, "o", "s"] -[644.530625, "o", "h"] -[644.580886, "o", " "] -[644.630903, "o", "b"] -[644.716123, "o", "o"] -[644.765843, "o", "o"] -[644.815942, "o", "t"] -[644.866147, "o", "s"] -[644.956244, "o", "t"] -[645.047474, "o", "r"] -[645.097416, "o", "a"] -[645.147532, "o", "p"] -[645.197732, "o", "p"] -[645.301883, "o", "i"] -[645.450908, "o", "n"] -[645.501035, "o", "g"] -[645.551045, "o", "."] -[645.601232, "o", "."] -[645.651242, "o", ".\r\n\u001b[?2004l\r"] -[645.651351, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[646.651618, "o", "s"] -[646.716658, "o", "l"] -[646.766956, "o", "e"] -[646.854055, "o", "e"] -[646.904428, "o", "p"] -[646.95438, "o", " "] -[647.004358, "o", "3"] -[647.199404, "o", "0"] -[647.371686, "o", "0\r\n\u001b[?2004l\r"] -[947.375079, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[947.375347, "o", "#"] -[947.425696, "o", " "] -[947.508576, "o", "S"] -[947.558675, "o", "t"] -[947.615774, "o", "e"] -[947.665865, "o", "p"] -[947.715991, "o", " "] -[947.76605, "o", "3"] -[947.816146, "o", ":"] -[947.866203, "o", " "] -[947.916309, "o", "C"] -[947.966356, "o", "o"] -[948.016516, "o", "n"] -[948.066533, "o", "n"] -[948.116689, "o", "e"] -[948.166858, "o", "c"] -[948.237809, "o", "t"] -[948.287885, "o", " "] -[948.337977, "o", "t"] -[948.416102, "o", "o"] -[948.466154, "o", " "] -[948.528228, "o", "C"] -[948.578418, "o", "o"] -[948.628566, "o", "n"] -[948.678825, "o", "s"] -[948.838579, "o", "t"] -[948.912696, "o", "e"] -[948.97679, "o", "l"] -[949.026892, "o", "l"] -[949.076916, "o", "a"] -[949.135133, "o", "t"] -[949.185214, "o", "i"] -[949.235581, "o", "o"] -[949.285484, "o", "n\r\n\u001b[?2004l\r"] -[949.285742, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[950.28631, "o", "e"] -[950.351409, "o", "x"] -[950.466544, "o", "p"] -[950.618724, "o", "o"] -[950.918755, "o", "r"] -[951.043786, "o", "t"] -[951.093986, "o", " "] -[951.143974, "o", "K"] -[951.194125, "o", "U"] -[951.244227, "o", "B"] -[951.294397, "o", "E"] -[951.344573, "o", "C"] -[951.394506, "o", "O"] -[951.444885, "o", "N"] -[951.494861, "o", "F"] -[951.544938, "o", "I"] -[951.628977, "o", "G"] -[951.711089, "o", "="] -[951.761344, "o", "/"] -[951.907458, "o", "c"] -[951.970367, "o", "o"] -[952.103606, "o", "n"] -[952.15961, "o", "s"] -[952.209672, "o", "t"] -[952.262769, "o", "e"] -[952.312798, "o", "l"] -[952.389873, "o", "l"] -[952.439985, "o", "a"] -[952.49011, "o", "t"] -[952.540451, "o", "i"] -[952.590326, "o", "o"] -[952.640429, "o", "n"] -[952.690535, "o", "/"] -[952.74067, "o", "c"] -[952.790832, "o", "o"] -[952.840824, "o", "n"] -[952.908826, "o", "s"] -[952.959, "o", "t"] -[953.009144, "o", "e"] -[953.05947, "o", "l"] -[953.109399, "o", "l"] -[953.176521, "o", "a"] -[953.226658, "o", "t"] -[953.276775, "o", "i"] -[953.347758, "o", "o"] -[953.505976, "o", "n"] -[953.556016, "o", "-"] -[953.606239, "o", "a"] -[953.656322, "o", "d"] -[953.710238, "o", "m"] -[953.760489, "o", "i"] -[953.810439, "o", "n"] -[953.860653, "o", "."] -[954.03074, "o", "c"] -[954.080676, "o", "o"] -[954.130816, "o", "n"] -[954.180925, "o", "f\r\n\u001b[?2004l\r"] -[954.181026, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[955.181355, "o", "k"] -[955.299397, "o", "u"] -[955.349463, "o", "b"] -[955.405546, "o", "e"] -[955.455605, "o", "c"] -[955.505827, "o", "t"] -[955.579909, "o", "l"] -[955.630037, "o", " "] -[955.696226, "o", "g"] -[955.747248, "o", "e"] -[955.849262, "o", "t"] -[955.899538, "o", " "] -[956.008494, "o", "n"] -[956.067586, "o", "o"] -[956.117594, "o", "d"] -[956.167734, "o", "e"] -[956.217826, "o", "s\r\n\u001b[?2004l\r"] -[956.413123, "o", "NAME STATUS ROLES AGE VERSION\r\nconstell-06f5aef5-control-plane-8248d5a2-98dd Ready control-plane 3m27s v1.27.7\r\nconstell-06f5aef5-control-plane-8248d5a2-bbmf Ready control-plane 7m42s v1.27.7\r\nconstell-06f5aef5-control-plane-8248d5a2-hxjj Ready control-plane 4m29s v1.27.7\r\nconstell-06f5aef5-worker-6d005d67-mgzx Ready 4m48s v1.27.7"] -[956.413184, "o", "\r\n"] -[956.415843, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[956.416052, "o", "exit"] +{"version": 2, "width": 199, "height": 20, "timestamp": 1703674791, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} +[0.003932, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[0.004442, "o", "#"] +[0.13627, "o", " "] +[0.186407, "o", "S"] +[0.236501, "o", "t"] +[0.307613, "o", "e"] +[0.357752, "o", "p"] +[0.407798, "o", " "] +[0.527922, "o", "1"] +[0.578048, "o", ":"] +[0.628181, "o", " "] +[0.678254, "o", "C"] +[0.785391, "o", "r"] +[0.835465, "o", "e"] +[0.917608, "o", "a"] +[0.967677, "o", "t"] +[1.017778, "o", "e"] +[1.037903, "o", " "] +[1.067903, "o", "t"] +[1.087903, "o", "h"] +[1.097903, "o", "e"] +[1.127903, "o", " "] +[1.143014, "o", "C"] +[1.232114, "o", "o"] +[1.282227, "o", "n"] +[1.406303, "o", "s"] +[1.456428, "o", "t"] +[1.517561, "o", "e"] +[1.62366, "o", "l"] +[1.6738, "o", "l"] +[1.723877, "o", "a"] +[1.77399, "o", "t"] +[1.824095, "o", "i"] +[1.87418, "o", "o"] +[1.924301, "o", "n"] +[1.97442, "o", " "] +[2.024526, "o", "c"] +[2.07459, "o", "l"] +[2.1247, "o", "u"] +[2.17482, "o", "s"] +[2.224927, "o", "t"] +[2.275007, "o", "e"] +[2.41116, "o", "r\r\n\u001b[?2004l\r"] +[2.411311, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[3.411738, "o", "c"] +[3.476772, "o", "o"] +[3.526845, "o", "n"] +[3.592982, "o", "s"] +[3.643096, "o", "t"] +[3.730205, "o", "e"] +[3.780291, "o", "l"] +[3.830406, "o", "l"] +[3.88851, "o", "a"] +[3.938607, "o", "t"] +[4.044725, "o", "i"] +[4.094826, "o", "o"] +[4.144943, "o", "n"] +[4.195041, "o", " "] +[4.296156, "o", "a"] +[4.346307, "o", "p"] +[4.402382, "o", "p"] +[4.520462, "o", "l"] +[4.602605, "o", "y\r\n\u001b[?2004l\r"] +[4.63371, "o", "Using community license.\r\n"] +[6.514164, "o", "For details, see https://docs.edgeless.systems/constellation/overview/license\r\n"] +[13.901035, "o", "The following Constellation cluster will be created:\r\n 3 control-plane nodes of type n2d-standard-4 will be created.\r\n 1 worker node of type n2d-standard-4 will be created.\r\nDo you want to create this cluster? [y/n]: "] +[13.901255, "o", "y\r\n"] +[180.722532, "o", "Cloud infrastructure created successfully.\r\n"] +[180.727688, "o", "Your Constellation master secret was successfully written to \"constellation-mastersecret.json\"\r\n"] +[618.109354, "o", "Your Constellation cluster was successfully initialized.\r\n\r\nConstellation cluster identifier 9eb7de42a641b8356176bb39e03d29a1dc7d1dab142cd1c39283cf329b53fe1f\r\nKubernetes configuration constellation-admin.conf\r\n\r\nYou can now connect to your cluster by executing:\r\n\texport KUBECONFIG=\"/constellation/constellation-admin.conf\"\r\n\r\n"] +[618.112273, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[618.112452, "o", "#"] +[618.162615, "o", " "] +[618.27972, "o", "W"] +[618.329771, "o", "a"] +[618.452971, "o", "i"] +[618.503067, "o", "t"] +[618.553173, "o", " "] +[618.659285, "o", "f"] +[618.717399, "o", "o"] +[618.789486, "o", "r"] +[618.839562, "o", " "] +[618.889693, "o", "c"] +[618.939774, "o", "l"] +[619.022877, "o", "u"] +[619.072989, "o", "s"] +[619.231143, "o", "t"] +[619.281216, "o", "e"] +[619.331289, "o", "r"] +[619.381425, "o", " "] +[619.431537, "o", "t"] +[619.486631, "o", "o"] +[619.536746, "o", " "] +[619.586851, "o", "f"] +[619.746988, "o", "i"] +[619.797078, "o", "n"] +[619.984209, "o", "i"] +[620.071254, "o", "s"] +[620.121392, "o", "h"] +[620.171525, "o", " "] +[620.221608, "o", "b"] +[620.271693, "o", "o"] +[620.321798, "o", "o"] +[620.371899, "o", "t"] +[620.422017, "o", "s"] +[620.543136, "o", "t"] +[620.593202, "o", "r"] +[620.643302, "o", "a"] +[620.6984, "o", "p"] +[620.748497, "o", "p"] +[620.849635, "o", "i"] +[620.89974, "o", "n"] +[620.949856, "o", "g"] +[620.999931, "o", "."] +[621.053046, "o", "."] +[621.128139, "o", ".\r\n\u001b[?2004l\r"] +[621.128243, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[622.128706, "o", "s"] +[622.178801, "o", "l"] +[622.245878, "o", "e"] +[622.295996, "o", "e"] +[622.34909, "o", "p"] +[622.399213, "o", " "] +[622.449322, "o", "3"] +[622.518402, "o", "0"] +[622.568555, "o", "0\r\n\u001b[?2004l\r"] +[922.569816, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[922.570091, "o", "#"] +[922.620252, "o", " "] +[922.781381, "o", "S"] +[922.831473, "o", "t"] +[922.881535, "o", "e"] +[922.931669, "o", "p"] +[922.981802, "o", " "] +[923.031901, "o", "2"] +[923.081984, "o", ":"] +[923.132097, "o", " "] +[923.182194, "o", "C"] +[923.232299, "o", "o"] +[923.282392, "o", "n"] +[923.344508, "o", "n"] +[923.394605, "o", "e"] +[923.444718, "o", "c"] +[923.494807, "o", "t"] +[923.544924, "o", " "] +[923.594999, "o", "t"] +[923.645113, "o", "o"] +[923.695213, "o", " "] +[923.745331, "o", "C"] +[923.826438, "o", "o"] +[923.876552, "o", "n"] +[923.960705, "o", "s"] +[924.010765, "o", "t"] +[924.095864, "o", "e"] +[924.145985, "o", "l"] +[924.1961, "o", "l"] +[924.246204, "o", "a"] +[924.33634, "o", "t"] +[924.427441, "o", "i"] +[924.477518, "o", "o"] +[924.527621, "o", "n\r\n\u001b[?2004l\r"] +[924.527698, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[925.528137, "o", "e"] +[925.632219, "o", "x"] +[925.781333, "o", "p"] +[925.831428, "o", "o"] +[925.881544, "o", "r"] +[925.931644, "o", "t"] +[925.981765, "o", " "] +[926.031898, "o", "K"] +[926.097042, "o", "U"] +[926.147111, "o", "B"] +[926.23429, "o", "E"] +[926.284331, "o", "C"] +[926.339442, "o", "O"] +[926.389539, "o", "N"] +[926.584675, "o", "F"] +[926.756799, "o", "I"] +[926.806872, "o", "G"] +[926.856936, "o", "="] +[926.907083, "o", "/"] +[926.957183, "o", "c"] +[927.014283, "o", "o"] +[927.06438, "o", "n"] +[927.114506, "o", "s"] +[927.164609, "o", "t"] +[927.214722, "o", "e"] +[927.26483, "o", "l"] +[927.31494, "o", "l"] +[927.36502, "o", "a"] +[927.415112, "o", "t"] +[927.465239, "o", "i"] +[927.515332, "o", "o"] +[927.565458, "o", "n"] +[927.615551, "o", "/"] +[927.665689, "o", "c"] +[927.715762, "o", "o"] +[927.793871, "o", "n"] +[927.84395, "o", "s"] +[927.906071, "o", "t"] +[927.956178, "o", "e"] +[928.006281, "o", "l"] +[928.056398, "o", "l"] +[928.216517, "o", "a"] +[928.290615, "o", "t"] +[928.354726, "o", "i"] +[928.404809, "o", "o"] +[928.454909, "o", "n"] +[928.505005, "o", "-"] +[928.555124, "o", "a"] +[928.605221, "o", "d"] +[928.655301, "o", "m"] +[928.705434, "o", "i"] +[928.770572, "o", "n"] +[928.820661, "o", "."] +[928.972754, "o", "c"] +[929.272921, "o", "o"] +[929.398058, "o", "n"] +[929.448135, "o", "f\r\n\u001b[?2004l\r"] +[929.448274, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[930.448699, "o", "k"] +[930.498765, "o", "u"] +[930.548887, "o", "b"] +[930.598986, "o", "e"] +[930.649151, "o", "c"] +[930.699174, "o", "t"] +[930.74929, "o", "l"] +[930.799366, "o", " "] +[930.849512, "o", "g"] +[930.933616, "o", "e"] +[931.015733, "o", "t"] +[931.065804, "o", " "] +[931.211941, "o", "n"] +[931.275028, "o", "o"] +[931.408148, "o", "d"] +[931.464248, "o", "e"] +[931.514347, "o", "s\r\n\u001b[?2004l\r"] +[932.37047, "o", "NAME STATUS ROLES AGE VERSION\r\nconstell-1088351e-control-plane-caac834e-dvf9 Ready control-plane 3m13s v1.27.8\r\nconstell-1088351e-control-plane-caac834e-rjq7 Ready control-plane 8m36s v1.27.8\r\nconstell-1088351e-control-plane-caac834e-w6fk Ready control-plane 4m26s v1.27.8\r\nconstell-1088351e-worker-5f94f70a-bf9d Ready 4m46s v1.27.8\r\n"] +[932.372893, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] diff --git a/docs/static/assets/terminate-cluster.cast b/docs/static/assets/terminate-cluster.cast index bd8ae6723..321005093 100644 --- a/docs/static/assets/terminate-cluster.cast +++ b/docs/static/assets/terminate-cluster.cast @@ -1,148 +1,146 @@ -{"version": 2, "width": 126, "height": 61, "timestamp": 1700562386, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} -[0.009553, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[0.010076, "o", "#"] -[0.141841, "o", " "] -[0.191981, "o", "S"] -[0.242115, "o", "t"] -[0.313199, "o", "e"] -[0.363326, "o", "p"] -[0.413499, "o", " "] -[0.533479, "o", "1"] -[0.583598, "o", ":"] -[0.633655, "o", " "] -[0.683789, "o", "D"] -[0.790838, "o", "e"] -[0.840914, "o", "l"] -[0.923077, "o", "e"] -[0.973285, "o", "t"] -[1.023282, "o", "e"] -[1.073612, "o", " "] -[1.148497, "o", "C"] -[1.237701, "o", "o"] -[1.287633, "o", "n"] -[1.411821, "o", "s"] -[1.461932, "o", "t"] -[1.523049, "o", "e"] -[1.629399, "o", "l"] -[1.679312, "o", "l"] -[1.729488, "o", "a"] -[1.779445, "o", "t"] -[1.829556, "o", "i"] -[1.879623, "o", "o"] -[1.929773, "o", "n"] -[1.979942, "o", " "] -[2.029897, "o", "c"] -[2.080079, "o", "l"] -[2.130337, "o", "u"] -[2.180291, "o", "s"] -[2.230385, "o", "t"] -[2.280459, "o", "e"] -[2.416593, "o", "r\r\n"] -[2.416686, "o", "\u001b[?2004l\r"] -[2.416758, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[3.416992, "o", "c"] -[3.482127, "o", "o"] -[3.532345, "o", "n"] -[3.598576, "o", "s"] -[3.648486, "o", "t"] -[3.735667, "o", "e"] -[3.785719, "o", "l"] -[3.835832, "o", "l"] -[3.893905, "o", "a"] -[3.944033, "o", "t"] -[4.050486, "o", "i"] -[4.100237, "o", "o"] -[4.150461, "o", "n"] -[4.200599, "o", " "] -[4.301521, "o", "t"] -[4.351605, "o", "e"] -[4.407697, "o", "r"] -[4.525781, "o", "m"] -[4.60797, "o", "i"] -[4.781962, "o", "n"] -[4.832108, "o", "a"] -[4.949299, "o", "t"] -[4.999348, "o", "e\r\n\u001b[?2004l\r"] -[5.081361, "o", "You are about to terminate a Constellation cluster.\r\nAll of its associated resources will be DESTROYED.\r\nThis action is irreversible and ALL DATA WILL BE LOST.\r\nDo you want to continue? [y/n]: "] -[6.081663, "o", "y\r\n"] -[207.531817, "o", "Your Constellation cluster was terminated successfully.\r\n"] -[207.535064, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[207.535291, "o", "#"] -[207.585776, "o", " "] -[207.826621, "o", "D"] -[207.932845, "o", "e"] -[207.991192, "o", "l"] -[208.063152, "o", "e"] -[208.113199, "o", "t"] -[208.163314, "o", "e"] -[208.213302, "o", " "] -[208.296371, "o", "m"] -[208.346781, "o", "a"] -[208.504906, "o", "s"] -[208.555108, "o", "t"] -[208.60499, "o", "e"] -[208.655416, "o", "r"] -[208.705324, "o", "s"] -[208.760525, "o", "e"] -[208.810626, "o", "c"] -[208.860687, "o", "r"] -[209.02053, "o", "e"] -[209.070683, "o", "t"] -[209.120727, "o", " "] -[209.207802, "o", "t"] -[209.257886, "o", "o"] -[209.307981, "o", " "] -[209.35824, "o", "f"] -[209.408476, "o", "i"] -[209.45844, "o", "n"] -[209.508808, "o", "a"] -[209.55868, "o", "l"] -[209.679619, "o", "i"] -[209.729745, "o", "z"] -[209.779862, "o", "e"] -[209.829937, "o", " "] -[209.880119, "o", "d"] -[209.981498, "o", "e"] -[210.031366, "o", "l"] -[210.081309, "o", "e"] -[210.13149, "o", "t"] -[210.184668, "o", "i"] -[210.259769, "o", "o"] -[210.311762, "o", "n\r\n\u001b[?2004l\r"] -[210.311856, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[211.312331, "o", "r"] -[211.379455, "o", "m"] -[211.429454, "o", " "] -[211.48243, "o", "c"] -[211.53272, "o", "o"] -[211.582846, "o", "n"] -[211.651923, "o", "s"] -[211.701981, "o", "t"] -[211.782121, "o", "e"] -[211.832271, "o", "l"] -[211.993236, "o", "l"] -[212.043282, "o", "a"] -[212.093313, "o", "t"] -[212.143391, "o", "i"] -[212.257432, "o", "o"] -[212.307561, "o", "n"] -[212.357642, "o", "-"] -[212.407723, "o", "m"] -[212.45778, "o", "a"] -[212.507954, "o", "s"] -[212.558056, "o", "t"] -[212.620269, "o", "e"] -[212.670234, "o", "r"] -[212.720303, "o", "s"] -[212.770875, "o", "e"] -[212.820813, "o", "c"] -[212.870695, "o", "r"] -[212.920805, "o", "e"] -[212.99487, "o", "t"] -[213.044931, "o", "."] -[213.12598, "o", "j"] -[213.176086, "o", "s"] -[213.260238, "o", "o"] -[213.31028, "o", "n\r\n"] -[213.310322, "o", "\u001b[?2004l\r"] -[213.313259, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +{"version": 2, "width": 199, "height": 20, "timestamp": 1703675724, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} +[0.003821, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[0.004444, "o", "#"] +[0.136229, "o", " "] +[0.186275, "o", "S"] +[0.236401, "o", "t"] +[0.307468, "o", "e"] +[0.357602, "o", "p"] +[0.407687, "o", " "] +[0.52779, "o", "1"] +[0.577939, "o", ":"] +[0.628003, "o", " "] +[0.678102, "o", "D"] +[0.785195, "o", "e"] +[0.835354, "o", "l"] +[0.917439, "o", "e"] +[0.967534, "o", "t"] +[1.017628, "o", "e"] +[1.067752, "o", " "] +[1.142842, "o", "C"] +[1.231966, "o", "o"] +[1.282062, "o", "n"] +[1.406167, "o", "s"] +[1.456274, "o", "t"] +[1.517366, "o", "e"] +[1.623448, "o", "l"] +[1.673552, "o", "l"] +[1.7236, "o", "a"] +[1.773715, "o", "t"] +[1.82384, "o", "i"] +[1.873926, "o", "o"] +[1.924014, "o", "n"] +[1.974102, "o", " "] +[2.024239, "o", "c"] +[2.074327, "o", "l"] +[2.124421, "o", "u"] +[2.174511, "o", "s"] +[2.224644, "o", "t"] +[2.274745, "o", "e"] +[2.410815, "o", "r\r\n\u001b[?2004l\r"] +[2.410919, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[3.411419, "o", "c"] +[3.47645, "o", "o"] +[3.526541, "o", "n"] +[3.592638, "o", "s"] +[3.642736, "o", "t"] +[3.729842, "o", "e"] +[3.779954, "o", "l"] +[3.830038, "o", "l"] +[3.888166, "o", "a"] +[3.938244, "o", "t"] +[4.044389, "o", "i"] +[4.094483, "o", "o"] +[4.144593, "o", "n"] +[4.194592, "o", " "] +[4.295774, "o", "t"] +[4.345884, "o", "e"] +[4.402033, "o", "r"] +[4.520134, "o", "m"] +[4.602195, "o", "i"] +[4.776347, "o", "n"] +[4.826422, "o", "a"] +[4.943547, "o", "t"] +[4.993655, "o", "e\r\n\u001b[?2004l\r"] +[5.022077, "o", "You are about to terminate a Constellation cluster.\r\nAll of its associated resources will be DESTROYED.\r\nThis action is irreversible and ALL DATA WILL BE LOST.\r\nDo you want to continue? [y/n]: "] +[6.022582, "o", "y\r\n"] +[220.660336, "o", "Your Constellation cluster was terminated successfully.\r\n"] +[220.661951, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[220.662116, "o", "#"] +[220.712361, "o", " "] +[220.953524, "o", "D"] +[221.059622, "o", "e"] +[221.117733, "o", "l"] +[221.189896, "o", "e"] +[221.23999, "o", "t"] +[221.290148, "o", "e"] +[221.340253, "o", " "] +[221.423394, "o", "m"] +[221.473505, "o", "a"] +[221.631651, "o", "s"] +[221.681726, "o", "t"] +[221.731891, "o", "e"] +[221.781934, "o", "r"] +[221.832052, "o", "s"] +[221.887179, "o", "e"] +[221.93733, "o", "c"] +[221.987438, "o", "r"] +[222.147636, "o", "e"] +[222.197733, "o", "t"] +[222.247788, "o", " "] +[222.334942, "o", "t"] +[222.385054, "o", "o"] +[222.435166, "o", " "] +[222.485299, "o", "f"] +[222.535455, "o", "i"] +[222.585591, "o", "n"] +[222.635745, "o", "a"] +[222.685791, "o", "l"] +[222.806975, "o", "i"] +[222.857105, "o", "z"] +[222.907205, "o", "e"] +[222.957314, "o", " "] +[223.007463, "o", "d"] +[223.10865, "o", "e"] +[223.158698, "o", "l"] +[223.208848, "o", "e"] +[223.259032, "o", "t"] +[223.312129, "o", "i"] +[223.387271, "o", "o"] +[223.439369, "o", "n\r\n\u001b[?2004l\r"] +[223.43943, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[224.439913, "o", "r"] +[224.50707, "o", "m"] +[224.557155, "o", " "] +[224.610303, "o", "c"] +[224.660429, "o", "o"] +[224.710521, "o", "n"] +[224.779669, "o", "s"] +[224.829762, "o", "t"] +[224.909953, "o", "e"] +[224.960083, "o", "l"] +[225.12122, "o", "l"] +[225.171332, "o", "a"] +[225.221447, "o", "t"] +[225.271551, "o", "i"] +[225.385705, "o", "o"] +[225.435837, "o", "n"] +[225.485957, "o", "-"] +[225.536071, "o", "m"] +[225.586194, "o", "a"] +[225.636264, "o", "s"] +[225.686388, "o", "t"] +[225.748544, "o", "e"] +[225.798641, "o", "r"] +[225.848791, "o", "s"] +[225.898912, "o", "e"] +[225.949039, "o", "c"] +[225.999149, "o", "r"] +[226.049294, "o", "e"] +[226.123411, "o", "t"] +[226.173527, "o", "."] +[226.254679, "o", "j"] +[226.30479, "o", "s"] +[226.388908, "o", "o"] +[226.439112, "o", "n\r\n\u001b[?2004l\r"] +[226.440068, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] diff --git a/docs/static/assets/verify-cli.cast b/docs/static/assets/verify-cli.cast index 467f9fa6c..d8b33e204 100644 --- a/docs/static/assets/verify-cli.cast +++ b/docs/static/assets/verify-cli.cast @@ -1,814 +1,813 @@ -{"version": 2, "width": 126, "height": 61, "timestamp": 1700561200, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} -[0.01145, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[0.012485, "o", "#"] -[0.143897, "o", " "] -[0.194227, "o", "S"] -[0.244196, "o", "t"] -[0.315326, "o", "e"] -[0.365498, "o", "p"] -[0.415543, "o", " "] -[0.535495, "o", "1"] -[0.585746, "o", ":"] -[0.635941, "o", " "] -[0.685856, "o", "I"] -[0.793037, "o", "n"] -[0.843195, "o", "s"] -[0.925245, "o", "t"] -[0.975367, "o", "a"] -[1.025601, "o", "l"] -[1.193639, "o", "l"] -[1.24374, "o", " "] -[1.332915, "o", "S"] -[1.38305, "o", "L"] -[1.507352, "o", "S"] -[1.557223, "o", "A"] -[1.607368, "o", " "] -[1.713474, "o", "v"] -[1.7636, "o", "e"] -[1.813719, "o", "r"] -[1.864021, "o", "i"] -[1.91424, "o", "f"] -[1.964329, "o", "i"] -[2.014351, "o", "e"] -[2.115575, "o", "r\r\n\u001b[?2004l\r"] -[2.115892, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[3.11632, "o", "c"] -[3.166504, "o", "u"] -[3.216746, "o", "r"] -[3.266684, "o", "l"] -[3.316876, "o", " "] -[3.366952, "o", "-"] -[3.503171, "o", "s"] -[3.560245, "o", "L"] -[3.625539, "o", "O"] -[3.675541, "o", " "] -[3.741648, "o", "h"] -[3.791883, "o", "t"] -[3.878752, "o", "t"] -[3.928943, "o", "p"] -[3.979065, "o", "s"] -[4.029194, "o", ":"] -[4.079255, "o", "/"] -[4.18555, "o", "/"] -[4.235618, "o", "g"] -[4.285869, "o", "i"] -[4.365914, "o", "t"] -[4.467123, "o", "h"] -[4.517078, "o", "u"] -[4.57327, "o", "b"] -[4.623333, "o", "."] -[4.705453, "o", "c"] -[4.879612, "o", "o"] -[4.929697, "o", "m"] -[4.979809, "o", "/"] -[5.029912, "o", "s"] -[5.15303, "o", "l"] -[5.203139, "o", "s"] -[5.444372, "o", "a"] -[5.494197, "o", "-"] -[5.552538, "o", "f"] -[5.624663, "o", "r"] -[5.674792, "o", "a"] -[5.724976, "o", "m"] -[5.774898, "o", "e"] -[5.858228, "o", "w"] -[5.9083, "o", "o"] -[6.066293, "o", "r"] -[6.116498, "o", "k"] -[6.166538, "o", "/"] -[6.21693, "o", "s"] -[6.26684, "o", "l"] -[6.322136, "o", "s"] -[6.372042, "o", "a"] -[6.4222, "o", "-"] -[6.582406, "o", "v"] -[6.632505, "o", "e"] -[6.819715, "o", "r"] -[6.906723, "o", "i"] -[6.956757, "o", "f"] -[7.007096, "o", "i"] -[7.057097, "o", "e"] -[7.107176, "o", "r"] -[7.157257, "o", "/"] -[7.207353, "o", "r"] -[7.257475, "o", "e"] -[7.378684, "o", "l"] -[7.428784, "o", "e"] -[7.479124, "o", "a"] -[7.534161, "o", "s"] -[7.584199, "o", "e"] -[7.685645, "o", "s"] -[7.735445, "o", "/"] -[7.785522, "o", "l"] -[7.835697, "o", "a"] -[7.888774, "o", "t"] -[7.963949, "o", "e"] -[8.016221, "o", "s"] -[8.06614, "o", "t"] -[8.116241, "o", "/"] -[8.166436, "o", "d"] -[8.219504, "o", "o"] -[8.269777, "o", "w"] -[8.319849, "o", "n"] -[8.388911, "o", "l"] -[8.439088, "o", "o"] -[8.519086, "o", "a"] -[8.56932, "o", "d"] -[8.61954, "o", "/"] -[8.669533, "o", "s"] -[8.719725, "o", "l"] -[8.769758, "o", "s"] -[8.884066, "o", "a"] -[8.933964, "o", "-"] -[9.009242, "o", "v"] -[9.059423, "o", "e"] -[9.109381, "o", "r"] -[9.159524, "o", "i"] -[9.209592, "o", "f"] -[9.271723, "o", "i"] -[9.321927, "o", "e"] -[9.372066, "o", "r"] -[9.422128, "o", "-"] -[9.47225, "o", "l"] -[9.522387, "o", "i"] -[9.572489, "o", "n"] -[9.646599, "o", "u"] -[9.696804, "o", "x"] -[9.746785, "o", "-"] -[9.796931, "o", "a"] -[9.881025, "o", "m"] -[9.93119, "o", "d"] -[10.016301, "o", "6"] -[10.066331, "o", "4\r\n\u001b[?2004l\r"] -[12.719851, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[12.72008, "o", "s"] -[12.770358, "o", "u"] -[12.860494, "o", "d"] -[12.951845, "o", "o"] -[13.001707, "o", " "] -[13.051914, "o", "i"] -[13.101985, "o", "n"] -[13.205973, "o", "s"] -[13.355302, "o", "t"] -[13.405408, "o", "a"] -[13.455551, "o", "l"] -[13.505775, "o", "l"] -[13.555975, "o", " "] -[13.605977, "o", "s"] -[13.670837, "o", "l"] -[13.721172, "o", "s"] -[13.80842, "o", "a"] -[13.858374, "o", "-"] -[13.913506, "o", "v"] -[13.963496, "o", "e"] -[14.158686, "o", "r"] -[14.330712, "o", "i"] -[14.380795, "o", "f"] -[14.430879, "o", "i"] -[14.513932, "o", "e"] -[14.564043, "o", "r"] -[14.6141, "o", "-"] -[14.664369, "o", "l"] -[14.714292, "o", "i"] -[14.76448, "o", "n"] -[14.81447, "o", "u"] -[14.864601, "o", "x"] -[14.914736, "o", "-"] -[14.964972, "o", "a"] -[15.015117, "o", "m"] -[15.06505, "o", "d"] -[15.11509, "o", "6"] -[15.165258, "o", "4"] -[15.215262, "o", " "] -[15.265294, "o", "/"] -[15.315386, "o", "u"] -[15.394138, "o", "s"] -[15.444006, "o", "r"] -[15.494104, "o", "/"] -[15.544111, "o", "l"] -[15.594155, "o", "o"] -[15.644231, "o", "c"] -[15.804329, "o", "a"] -[15.878494, "o", "l"] -[15.928528, "o", "/"] -[15.97869, "o", "b"] -[16.028922, "o", "i"] -[16.087133, "o", "n"] -[16.137122, "o", "/"] -[16.187138, "o", "s"] -[16.237245, "o", "l"] -[16.287247, "o", "s"] -[16.352371, "o", "a"] -[16.402496, "o", "-"] -[16.554629, "o", "v"] -[16.854708, "o", "e"] -[16.979979, "o", "r"] -[17.029917, "o", "i"] -[17.08003, "o", "f"] -[17.130112, "o", "i"] -[17.180254, "o", "e"] -[17.230343, "o", "r\r\n\u001b[?2004l\r"] -[17.276874, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[18.27771, "o", "#"] -[18.327728, "o", " "] -[18.37792, "o", "S"] -[18.42801, "o", "t"] -[18.478255, "o", "e"] -[18.562251, "o", "p"] -[18.61235, "o", " "] -[18.717448, "o", "2"] -[18.767521, "o", ":"] -[18.830662, "o", " "] -[18.963844, "o", "D"] -[19.020054, "o", "o"] -[19.070002, "o", "w"] -[19.123267, "o", "n"] -[19.173154, "o", "l"] -[19.250258, "o", "o"] -[19.300265, "o", "a"] -[19.35038, "o", "d"] -[19.40055, "o", " "] -[19.450758, "o", "C"] -[19.500698, "o", "o"] -[19.551039, "o", "n"] -[19.601155, "o", "s"] -[19.65118, "o", "t"] -[19.701347, "o", "e"] -[19.769483, "o", "l"] -[19.819638, "o", "l"] -[19.869648, "o", "a"] -[19.9199, "o", "t"] -[19.970004, "o", "i"] -[20.037144, "o", "o"] -[20.087235, "o", "n"] -[20.137347, "o", " "] -[20.208456, "o", "C"] -[20.366626, "o", "L"] -[20.443666, "o", "I"] -[20.49381, "o", " "] -[20.543913, "o", "a"] -[20.598005, "o", "n"] -[20.648001, "o", "d"] -[20.69816, "o", " "] -[20.74826, "o", "p"] -[20.91828, "o", "r"] -[20.968503, "o", "o"] -[21.01856, "o", "v"] -[21.068622, "o", "e"] -[21.118864, "o", "n"] -[21.236861, "o", "a"] -[21.286976, "o", "n"] -[21.343188, "o", "c"] -[21.393257, "o", "e\r\n"] -[21.393524, "o", "\u001b[?2004l\r\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[22.393981, "o", "c"] -[22.46809, "o", "u"] -[22.518205, "o", "r"] -[22.584276, "o", "l"] -[22.634482, "o", " "] -[22.736579, "o", "-"] -[22.80961, "o", "s"] -[22.918757, "o", "L"] -[22.977962, "o", "O"] -[23.028013, "o", " "] -[23.0782, "o", "h"] -[23.128177, "o", "t"] -[23.199479, "o", "t"] -[23.249602, "o", "p"] -[23.446755, "o", "s"] -[23.496845, "o", ":"] -[23.56321, "o", "/"] -[23.613215, "o", "/"] -[23.827146, "o", "g"] -[23.877246, "o", "i"] -[23.927325, "o", "t"] -[23.977414, "o", "h"] -[24.04864, "o", "u"] -[24.098944, "o", "b"] -[24.149068, "o", "."] -[24.198988, "o", "c"] -[24.314329, "o", "o"] -[24.364147, "o", "m"] -[24.414201, "o", "/"] -[24.464459, "o", "e"] -[24.514594, "o", "d"] -[24.564673, "o", "g"] -[24.614927, "o", "e"] -[24.665062, "o", "l"] -[24.715237, "o", "e"] -[24.765328, "o", "s"] -[24.815517, "o", "s"] -[24.86554, "o", "s"] -[24.915664, "o", "y"] -[24.996945, "o", "s"] -[25.047108, "o", "/"] -[25.131215, "o", "c"] -[25.218152, "o", "o"] -[25.268054, "o", "n"] -[25.450315, "o", "s"] -[25.532438, "o", "t"] -[25.602475, "o", "e"] -[25.652672, "o", "l"] -[25.702732, "o", "l"] -[25.75281, "o", "a"] -[25.803142, "o", "t"] -[25.853199, "o", "i"] -[25.95224, "o", "o"] -[26.002379, "o", "n"] -[26.052465, "o", "/"] -[26.102536, "o", "r"] -[26.152622, "o", "e"] -[26.202684, "o", "l"] -[26.252838, "o", "e"] -[26.302934, "o", "a"] -[26.353169, "o", "s"] -[26.403227, "o", "e"] -[26.453248, "o", "s"] -[26.503472, "o", "/"] -[26.553546, "o", "l"] -[26.609667, "o", "a"] -[26.722758, "o", "t"] -[26.772957, "o", "e"] -[26.842207, "o", "s"] -[26.892221, "o", "t"] -[26.942379, "o", "/"] -[26.992558, "o", "d"] -[27.10661, "o", "o"] -[27.163739, "o", "w"] -[27.213868, "o", "n"] -[27.265114, "o", "l"] -[27.315215, "o", "o"] -[27.4432, "o", "a"] -[27.55344, "o", "d"] -[27.603345, "o", "/"] -[27.65352, "o", "c"] -[27.703638, "o", "o"] -[27.753739, "o", "n"] -[27.803901, "o", "s"] -[27.974992, "o", "t"] -[28.047101, "o", "e"] -[28.097273, "o", "l"] -[28.147384, "o", "l"] -[28.263489, "o", "a"] -[28.379663, "o", "t"] -[28.429777, "o", "i"] -[28.479841, "o", "o"] -[28.529909, "o", "n"] -[28.579918, "o", "-"] -[28.648416, "o", "l"] -[28.698307, "o", "i"] -[28.748456, "o", "n"] -[28.799435, "o", "u"] -[28.849518, "o", "x"] -[28.899601, "o", "-"] -[28.964892, "o", "a"] -[29.015088, "o", "m"] -[29.06513, "o", "d"] -[29.115515, "o", "6"] -[29.165319, "o", "4\r\n\u001b[?2004l\r"] -[31.611036, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[31.611172, "o", "c"] -[31.694348, "o", "u"] -[31.745447, "o", "r"] -[31.795656, "o", "l"] -[31.845768, "o", " "] -[31.919025, "o", "-"] -[31.981936, "o", "s"] -[32.032299, "o", "L"] -[32.082333, "o", "O"] -[32.132252, "o", " "] -[32.182349, "o", "h"] -[32.274414, "o", "t"] -[32.324504, "o", "t"] -[32.381801, "o", "p"] -[32.449982, "o", "s"] -[32.500042, "o", ":"] -[32.550239, "o", "/"] -[32.600278, "o", "/"] -[32.650517, "o", "g"] -[32.705463, "o", "i"] -[32.808719, "o", "t"] -[32.858853, "o", "h"] -[32.981712, "o", "u"] -[33.031804, "o", "b"] -[33.08216, "o", "."] -[33.203264, "o", "c"] -[33.280135, "o", "o"] -[33.330232, "o", "m"] -[33.38046, "o", "/"] -[33.430635, "o", "e"] -[33.480617, "o", "d"] -[33.530753, "o", "g"] -[33.603814, "o", "e"] -[33.658085, "o", "l"] -[33.726185, "o", "e"] -[33.776184, "o", "s"] -[33.826165, "o", "s"] -[33.876343, "o", "s"] -[33.979441, "o", "y"] -[34.029586, "o", "s"] -[34.079729, "o", "/"] -[34.129786, "o", "c"] -[34.179941, "o", "o"] -[34.230033, "o", "n"] -[34.364119, "o", "s"] -[34.414397, "o", "t"] -[34.464313, "o", "e"] -[34.546587, "o", "l"] -[34.768701, "o", "l"] -[34.821034, "o", "a"] -[34.870755, "o", "t"] -[34.921187, "o", "i"] -[35.006081, "o", "o"] -[35.056315, "o", "n"] -[35.106325, "o", "/"] -[35.156594, "o", "r"] -[35.206691, "o", "e"] -[35.256682, "o", "l"] -[35.384915, "o", "e"] -[35.435065, "o", "a"] -[35.489084, "o", "s"] -[35.629228, "o", "e"] -[35.681282, "o", "s"] -[35.731435, "o", "/"] -[35.781658, "o", "l"] -[35.831687, "o", "a"] -[35.88761, "o", "t"] -[35.968608, "o", "e"] -[36.018386, "o", "s"] -[36.068407, "o", "t"] -[36.118631, "o", "/"] -[36.168668, "o", "d"] -[36.246756, "o", "o"] -[36.297116, "o", "w"] -[36.347152, "o", "n"] -[36.411071, "o", "l"] -[36.461203, "o", "o"] -[36.514196, "o", "a"] -[36.56437, "o", "d"] -[36.6144, "o", "/"] -[36.664688, "o", "c"] -[36.740767, "o", "o"] -[36.797854, "o", "n"] -[36.912004, "o", "s"] -[37.065208, "o", "t"] -[37.115246, "o", "e"] -[37.165156, "o", "l"] -[37.215341, "o", "l"] -[37.310655, "o", "a"] -[37.417845, "o", "t"] -[37.467974, "o", "i"] -[37.518069, "o", "o"] -[37.604179, "o", "n"] -[37.654198, "o", "."] -[37.704482, "o", "i"] -[37.754398, "o", "n"] -[37.804623, "o", "t"] -[37.854791, "o", "o"] -[37.904697, "o", "t"] -[37.955053, "o", "o"] -[38.005193, "o", "."] -[38.055351, "o", "j"] -[38.105268, "o", "s"] -[38.155337, "o", "o"] -[38.22359, "o", "n"] -[38.273657, "o", "l\r\n\u001b[?2004l\r"] -[39.287345, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[39.287642, "o", "#"] -[39.33799, "o", " "] -[39.523106, "o", "S"] -[39.573122, "o", "t"] -[39.623296, "o", "e"] -[39.673316, "o", "p"] -[39.72358, "o", " "] -[39.798672, "o", "3"] -[39.848791, "o", ":"] -[39.898859, "o", " "] -[40.027149, "o", "V"] -[40.14818, "o", "e"] -[40.19822, "o", "r"] -[40.272299, "o", "i"] -[40.350546, "o", "f"] -[40.436577, "o", "y"] -[40.486706, "o", " "] -[40.536931, "o", "p"] -[40.587034, "o", "r"] -[40.637286, "o", "o"] -[40.720046, "o", "v"] -[40.770266, "o", "e"] -[40.820305, "o", "n"] -[40.902413, "o", "a"] -[40.959579, "o", "n"] -[41.022716, "o", "c"] -[41.07285, "o", "e\r\n\u001b[?2004l\r"] -[41.073081, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[42.073449, "o", "s"] -[42.143551, "o", "l"] -[42.193714, "o", "s"] -[42.24381, "o", "a"] -[42.294263, "o", "-"] -[42.350266, "o", "v"] -[42.400245, "o", "e"] -[42.450391, "o", "r"] -[42.500573, "o", "i"] -[42.657881, "o", "f"] -[42.828976, "o", "i"] -[42.87893, "o", "e"] -[42.935089, "o", "r"] -[42.985071, "o", " "] -[43.035352, "o", "v"] -[43.085266, "o", "e"] -[43.13557, "o", "r"] -[43.212658, "o", "i"] -[43.262784, "o", "f"] -[43.329018, "o", "y"] -[43.379143, "o", "-"] -[43.429139, "o", "a"] -[43.479628, "o", "r"] -[43.612486, "o", "t"] -[43.662623, "o", "i"] -[43.712737, "o", "f"] -[43.762925, "o", "a"] -[43.894199, "o", "c"] -[44.008282, "o", "t"] -[44.058114, "o", " "] -[44.143441, "o", "c"] -[44.210624, "o", "o"] -[44.260446, "o", "n"] -[44.310707, "o", "s"] -[44.424747, "o", "t"] -[44.474746, "o", "e"] -[44.587963, "o", "l"] -[44.668136, "o", "l"] -[44.718393, "o", "a"] -[44.768428, "o", "t"] -[44.838611, "o", "i"] -[44.888677, "o", "o"] -[44.938783, "o", "n"] -[44.98892, "o", "-"] -[45.039001, "o", "l"] -[45.089357, "o", "i"] -[45.139219, "o", "n"] -[45.189275, "o", "u"] -[45.239478, "o", "x"] -[45.289576, "o", "-"] -[45.339747, "o", "a"] -[45.401025, "o", "m"] -[45.480088, "o", "d"] -[45.530131, "o", "6"] -[45.580213, "o", "4"] -[45.630337, "o", " "] -[45.680415, "o", "-"] -[45.730697, "o", "-"] -[45.780801, "o", "p"] -[45.830954, "o", "r"] -[45.906079, "o", "o"] -[45.956164, "o", "v"] -[46.058296, "o", "e"] -[46.123253, "o", "n"] -[46.173232, "o", "a"] -[46.223432, "o", "n"] -[46.395533, "o", "c"] -[46.445743, "o", "e"] -[46.496018, "o", "-"] -[46.54607, "o", "p"] -[46.596134, "o", "a"] -[46.697273, "o", "t"] -[46.747318, "o", "h"] -[46.797389, "o", " "] -[46.847631, "o", "c"] -[46.931734, "o", "o"] -[46.981842, "o", "n"] -[47.031956, "o", "s"] -[47.082225, "o", "t"] -[47.132345, "o", "e"] -[47.18744, "o", "l"] -[47.237357, "o", "l"] -[47.370428, "o", "a"] -[47.420507, "o", "t"] -[47.470565, "o", "i"] -[47.520836, "o", "o"] -[47.570893, "o", "n"] -[47.621042, "o", "."] -[47.671151, "o", "i"] -[47.721226, "o", "n"] -[47.771281, "o", "t"] -[47.821402, "o", "o"] -[47.8715, "o", "t"] -[47.925537, "o", "o"] -[47.975602, "o", "."] -[48.106815, "o", "j"] -[48.156908, "o", "s"] -[48.218043, "o", "o"] -[48.268234, "o", "n"] -[48.318271, "o", "l"] -[48.368376, "o", " "] -[48.418506, "o", "-"] -[48.468736, "o", "-"] -[48.518949, "o", "s"] -[48.633019, "o", "o"] -[48.683022, "o", "u"] -[48.733202, "o", "r"] -[48.783189, "o", "c"] -[48.833439, "o", "e"] -[48.883359, "o", "-"] -[48.933445, "o", "u"] -[49.035662, "o", "r"] -[49.085857, "o", "i"] -[49.135963, "o", " "] -[49.186287, "o", "g"] -[49.259109, "o", "i"] -[49.309201, "o", "t"] -[49.359375, "o", "h"] -[49.409548, "o", "u"] -[49.459702, "o", "b"] -[49.509835, "o", "."] -[49.559907, "o", "c"] -[49.610029, "o", "o"] -[49.700187, "o", "m \r"] -[49.750371, "o", "/"] -[49.832515, "o", "e"] -[49.882637, "o", "d"] -[49.982745, "o", "g"] -[50.032739, "o", "e"] -[50.082842, "o", "l"] -[50.133068, "o", "e"] -[50.230156, "o", "s"] -[50.280075, "o", "s"] -[50.330149, "o", "s"] -[50.380351, "o", "y"] -[50.430366, "o", "s"] -[50.480476, "o", "/"] -[50.583727, "o", "c"] -[50.633784, "o", "o"] -[50.684074, "o", "n"] -[50.764978, "o", "s"] -[50.815275, "o", "t"] -[50.865384, "o", "e"] -[50.915399, "o", "l"] -[50.965487, "o", "l"] -[51.069693, "o", "a"] -[51.127701, "o", "t"] -[51.177746, "o", "i"] -[51.293996, "o", "o"] -[51.34394, "o", "n\r\n\u001b[?2004l\r"] -[52.558694, "o", "Verified signature against tlog entry index 49747663 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77afd31550fc3f5fb69e285d1ebb57f76d52dc7679627197e6cbcfc9d91d8e10a57\r\n"] -[52.569863, "o", "Verified build using builder \"https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.9.0\" at commit 246b9ce069acb0933ea8d2c786383b3e46d26936\r\nVerifying artifact constellation-linux-amd64: PASSED\r\n\r\nPASSED: Verified SLSA provenance\r\n"] -[52.572013, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[52.572161, "o", "#"] -[52.622428, "o", " "] -[52.672706, "o", "S"] -[52.72267, "o", "t"] -[52.812979, "o", "e"] -[52.863233, "o", "p"] -[52.913038, "o", " "] -[52.963176, "o", "4"] -[53.013224, "o", ":"] -[53.108309, "o", " "] -[53.158335, "o", "I"] -[53.246485, "o", "n"] -[53.30775, "o", "s"] -[53.404007, "o", "t"] -[53.565114, "o", "a"] -[53.614918, "o", "l"] -[53.740978, "o", "l"] -[53.791069, "o", " "] -[53.841366, "o", "t"] -[53.891316, "o", "h"] -[53.941423, "o", "e"] -[53.991573, "o", " "] -[54.041697, "o", "C"] -[54.091813, "o", "L"] -[54.217163, "o", "I\r\n\u001b[?2004l\r"] -[54.217393, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[55.217805, "o", "s"] -[55.272005, "o", "u"] -[55.32204, "o", "d"] -[55.372211, "o", "o"] -[55.422368, "o", " "] -[55.472441, "o", "i"] -[55.522638, "o", "n"] -[55.602528, "o", "s"] -[55.652701, "o", "t"] -[55.703055, "o", "a"] -[55.752939, "o", "l"] -[55.802945, "o", "l"] -[55.853056, "o", " "] -[55.927132, "o", "c"] -[55.977268, "o", "o"] -[56.065391, "o", "n"] -[56.115615, "o", "s"] -[56.217621, "o", "t"] -[56.267824, "o", "e"] -[56.364066, "o", "l"] -[56.453175, "o", "l"] -[56.552208, "o", "a"] -[56.626438, "o", "t"] -[56.68945, "o", "i"] -[56.739628, "o", "o"] -[56.78968, "o", "n"] -[56.839867, "o", "-"] -[56.890058, "o", "l"] -[57.006089, "o", "i"] -[57.056289, "o", "n"] -[57.255432, "o", "u"] -[57.305463, "o", "x"] -[57.355619, "o", "-"] -[57.405687, "o", "a"] -[57.455861, "o", "m"] -[57.50611, "o", "d"] -[57.602209, "o", "6"] -[57.652281, "o", "4"] -[57.702453, "o", " "] -[57.752392, "o", "/"] -[57.802485, "o", "u"] -[57.852589, "o", "s"] -[57.902613, "o", "r"] -[57.952777, "o", "/"] -[58.002938, "o", "l"] -[58.05909, "o", "o"] -[58.146327, "o", "c"] -[58.196347, "o", "a"] -[58.24647, "o", "l"] -[58.296595, "o", "/"] -[58.409607, "o", "b"] -[58.537958, "o", "i"] -[58.838039, "o", "n"] -[58.888136, "o", "/"] -[58.939131, "o", "c"] -[58.989333, "o", "o"] -[59.272415, "o", "n"] -[59.322497, "o", "s"] -[59.434651, "o", "t"] -[59.48986, "o", "e"] -[59.539953, "o", "l"] -[59.590981, "o", "l"] -[59.657207, "o", "a"] -[59.707165, "o", "t"] -[59.757517, "o", "i"] -[59.847638, "o", "o"] -[59.975691, "o", "n\r\n\u001b[?2004l\r"] -[60.028585, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[61.028998, "o", "#"] -[61.079109, "o", " "] -[61.12921, "o", "D"] -[61.228483, "o", "o"] -[61.278506, "o", "n"] -[61.328493, "o", "e"] -[61.378695, "o", "!"] -[61.466747, "o", " "] -[61.516737, "o", "Y"] -[61.579921, "o", "o"] -[61.633097, "o", "u"] -[61.683088, "o", " "] -[61.733186, "o", "c"] -[61.783387, "o", "a"] -[61.833466, "o", "n"] -[61.883642, "o", " "] -[61.93368, "o", "n"] -[61.993919, "o", "o"] -[62.044025, "o", "w"] -[62.094198, "o", " "] -[62.153135, "o", "u"] -[62.203263, "o", "s"] -[62.253346, "o", "e"] -[62.303498, "o", " "] -[62.368564, "o", "t"] -[62.427771, "o", "h"] -[62.528928, "o", "e"] -[62.579108, "o", " "] -[62.629025, "o", "v"] -[62.68925, "o", "e"] -[62.739202, "o", "r"] -[62.838374, "o", "i"] -[62.888479, "o", "f"] -[62.979775, "o", "i"] -[63.029607, "o", "e"] -[63.224156, "o", "d"] -[63.274001, "o", " "] -[63.346466, "o", "C"] -[63.429217, "o", "L"] -[63.548371, "o", "I\r\n"] -[63.54862, "o", "\u001b[?2004l\r"] -[63.548723, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] -[64.549292, "o", "c"] -[64.599545, "o", "o"] -[64.649488, "o", "n"] -[64.699697, "o", "s"] -[64.778756, "o", "t"] -[64.85084, "o", "e"] -[64.901016, "o", "l"] -[64.951119, "o", "l"] -[65.001402, "o", "a"] -[65.051326, "o", "t"] -[65.101593, "o", "i"] -[65.161598, "o", "o"] -[65.218847, "o", "n"] -[65.26897, "o", " "] -[65.359082, "o", "-"] -[65.409149, "o", "h\r\n\u001b[?2004l\r"] -[65.4415, "o", "Manage your Constellation cluster.\r\n\r\n"] -[65.441788, "o", "Usage:\r\n constellation [command]\r\n\r\nAvailable Commands:\r\n config Work with the Constellation configuration file\r\n create Create instances on a cloud platform for your Constellation cluster\r\n apply Apply a configuration to a Constellation cluster\r\n mini Manage MiniConstellation clusters\r\n status Show status of a Constellation cluster\r\n verify Verify the confidential properties of a Constellation cluster\r\n upgrade Find and apply upgrades to your Constellation cluster\r\n recover Recover a completely stopped Constellation cluster\r\n terminate Terminate a Constellation cluster\r\n iam Work with the IAM configuration on your cloud provider\r\n version Display version of this CLI\r\n help Help about any command\r\n completion Generate the autocompletion script for the specified shell\r\n\r\nFlags:\r\n --debug enable debug logging\r\n --force disable version compatibility checks - might result in corrupted clusters\r\n -h, -"] -[65.441928, "o", "-help help for constellation\r\n --tf-log string Terraform log level (default \"NONE\")\r\n -C, --workspace string path to the Constellation workspace\r\n\r\nUse \"constellation [command] --help\" for more information about a command.\r\n"] -[65.443864, "o", "\u001b[?2004h"] -[65.443943, "o", "\u001b[38;2;139;4;221m$\u001b[0m "] +{"version": 2, "width": 199, "height": 20, "timestamp": 1703674587, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} +[0.004036, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[0.00454, "o", "#"] +[0.136422, "o", " "] +[0.186496, "o", "S"] +[0.236643, "o", "t"] +[0.307752, "o", "e"] +[0.357809, "o", "p"] +[0.408009, "o", " "] +[0.528038, "o", "1"] +[0.57813, "o", ":"] +[0.628284, "o", " "] +[0.678373, "o", "I"] +[0.785492, "o", "n"] +[0.835591, "o", "s"] +[0.917712, "o", "t"] +[0.967809, "o", "a"] +[1.017919, "o", "l"] +[1.186017, "o", "l"] +[1.236136, "o", " "] +[1.325214, "o", "S"] +[1.375349, "o", "L"] +[1.499453, "o", "S"] +[1.549549, "o", "A"] +[1.599663, "o", " "] +[1.705732, "o", "v"] +[1.755864, "o", "e"] +[1.805969, "o", "r"] +[1.856055, "o", "i"] +[1.906194, "o", "f"] +[1.956289, "o", "i"] +[2.006401, "o", "e"] +[2.107494, "o", "r\r\n\u001b[?2004l\r"] +[2.107605, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[3.108, "o", "c"] +[3.158074, "o", "u"] +[3.208205, "o", "r"] +[3.258285, "o", "l"] +[3.308417, "o", " "] +[3.358507, "o", "-"] +[3.494607, "o", "s"] +[3.55172, "o", "L"] +[3.616839, "o", "O"] +[3.666961, "o", " "] +[3.733047, "o", "h"] +[3.783076, "o", "t"] +[3.870249, "o", "t"] +[3.920355, "o", "p"] +[3.970469, "o", "s"] +[4.020549, "o", ":"] +[4.070675, "o", "/"] +[4.176737, "o", "/"] +[4.226839, "o", "g"] +[4.276971, "o", "i"] +[4.357024, "o", "t"] +[4.458172, "o", "h"] +[4.508288, "o", "u"] +[4.564381, "o", "b"] +[4.614455, "o", "."] +[4.696596, "o", "c"] +[4.870675, "o", "o"] +[4.920807, "o", "m"] +[4.970913, "o", "/"] +[5.020971, "o", "s"] +[5.144117, "o", "l"] +[5.194246, "o", "s"] +[5.43533, "o", "a"] +[5.485447, "o", "-"] +[5.543557, "o", "f"] +[5.615647, "o", "r"] +[5.665764, "o", "a"] +[5.715849, "o", "m"] +[5.765951, "o", "e"] +[5.849046, "o", "w"] +[5.899178, "o", "o"] +[6.05732, "o", "r"] +[6.107387, "o", "k"] +[6.15751, "o", "/"] +[6.207618, "o", "s"] +[6.257724, "o", "l"] +[6.312833, "o", "s"] +[6.362891, "o", "a"] +[6.413029, "o", "-"] +[6.573163, "o", "v"] +[6.623248, "o", "e"] +[6.810385, "o", "r"] +[6.897478, "o", "i"] +[6.947609, "o", "f"] +[6.997679, "o", "i"] +[7.047814, "o", "e"] +[7.097888, "o", "r"] +[7.148021, "o", "/"] +[7.198087, "o", "r"] +[7.248222, "o", "e"] +[7.369284, "o", "l"] +[7.419395, "o", "e"] +[7.469527, "o", "a"] +[7.524686, "o", "s"] +[7.57473, "o", "e"] +[7.675831, "o", "s"] +[7.725926, "o", "/"] +[7.776038, "o", "l"] +[7.826123, "o", "a"] +[7.879235, "o", "t"] +[7.954339, "o", "e"] +[8.006397, "o", "s"] +[8.05655, "o", "t"] +[8.106625, "o", "/"] +[8.156743, "o", "d"] +[8.209861, "o", "o"] +[8.259954, "o", "w"] +[8.310043, "o", "n"] +[8.379166, "o", "l"] +[8.429264, "o", "o"] +[8.509405, "o", "a"] +[8.559447, "o", "d"] +[8.609562, "o", "/"] +[8.659671, "o", "s"] +[8.709764, "o", "l"] +[8.759921, "o", "s"] +[8.874, "o", "a"] +[8.924102, "o", "-"] +[8.99921, "o", "v"] +[9.049314, "o", "e"] +[9.099402, "o", "r"] +[9.14952, "o", "i"] +[9.199612, "o", "f"] +[9.261734, "o", "i"] +[9.311807, "o", "e"] +[9.361942, "o", "r"] +[9.41203, "o", "-"] +[9.462148, "o", "l"] +[9.512254, "o", "i"] +[9.562358, "o", "n"] +[9.636469, "o", "u"] +[9.686562, "o", "x"] +[9.736677, "o", "-"] +[9.786778, "o", "a"] +[9.870888, "o", "m"] +[9.920992, "o", "d"] +[10.006102, "o", "6"] +[10.056228, "o", "4\r\n\u001b[?2004l\r"] +[10.83884, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[10.839099, "o", "s"] +[10.889223, "o", "u"] +[10.979329, "o", "d"] +[11.070438, "o", "o"] +[11.12053, "o", " "] +[11.170659, "o", "i"] +[11.220766, "o", "n"] +[11.32484, "o", "s"] +[11.474003, "o", "t"] +[11.524092, "o", "a"] +[11.574165, "o", "l"] +[11.624263, "o", "l"] +[11.674289, "o", " "] +[11.724447, "o", "s"] +[11.789571, "o", "l"] +[11.839663, "o", "s"] +[11.926776, "o", "a"] +[11.976882, "o", "-"] +[12.031992, "o", "v"] +[12.082066, "o", "e"] +[12.277215, "o", "r"] +[12.449313, "o", "i"] +[12.499416, "o", "f"] +[12.549512, "o", "i"] +[12.632595, "o", "e"] +[12.682632, "o", "r"] +[12.732789, "o", "-"] +[12.782884, "o", "l"] +[12.833024, "o", "i"] +[12.883122, "o", "n"] +[12.933216, "o", "u"] +[12.983311, "o", "x"] +[13.033429, "o", "-"] +[13.083541, "o", "a"] +[13.13365, "o", "m"] +[13.183742, "o", "d"] +[13.23386, "o", "6"] +[13.283962, "o", "4"] +[13.33407, "o", " "] +[13.384178, "o", "/"] +[13.434286, "o", "u"] +[13.512371, "o", "s"] +[13.562449, "o", "r"] +[13.612534, "o", "/"] +[13.662662, "o", "l"] +[13.712779, "o", "o"] +[13.762857, "o", "c"] +[13.922948, "o", "a"] +[13.997092, "o", "l"] +[14.047204, "o", "/"] +[14.097293, "o", "b"] +[14.147407, "o", "i"] +[14.205567, "o", "n"] +[14.255645, "o", "/"] +[14.305748, "o", "s"] +[14.355855, "o", "l"] +[14.405975, "o", "s"] +[14.471063, "o", "a"] +[14.521178, "o", "-"] +[14.673229, "o", "v"] +[14.973393, "o", "e"] +[15.098469, "o", "r"] +[15.148599, "o", "i"] +[15.198681, "o", "f"] +[15.24878, "o", "i"] +[15.298884, "o", "e"] +[15.349042, "o", "r\r\n\u001b[?2004l\r"] +[15.372162, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[16.372653, "o", "#"] +[16.422657, "o", " "] +[16.472805, "o", "S"] +[16.522862, "o", "t"] +[16.573003, "o", "e"] +[16.657011, "o", "p"] +[16.707185, "o", " "] +[16.81232, "o", "2"] +[16.862426, "o", ":"] +[16.925516, "o", " "] +[17.058634, "o", "D"] +[17.1147, "o", "o"] +[17.164812, "o", "w"] +[17.217931, "o", "n"] +[17.268026, "o", "l"] +[17.345167, "o", "o"] +[17.395202, "o", "a"] +[17.445343, "o", "d"] +[17.495448, "o", " "] +[17.545545, "o", "C"] +[17.595635, "o", "o"] +[17.645803, "o", "n"] +[17.69587, "o", "s"] +[17.746007, "o", "t"] +[17.796087, "o", "e"] +[17.864206, "o", "l"] +[17.914215, "o", "l"] +[17.964343, "o", "a"] +[18.014421, "o", "t"] +[18.064553, "o", "i"] +[18.131661, "o", "o"] +[18.181755, "o", "n"] +[18.231876, "o", " "] +[18.302949, "o", "C"] +[18.461091, "o", "L"] +[18.538175, "o", "I"] +[18.588264, "o", " "] +[18.638349, "o", "a"] +[18.692454, "o", "n"] +[18.742563, "o", "d"] +[18.792665, "o", " "] +[18.842766, "o", "p"] +[19.0129, "o", "r"] +[19.062989, "o", "o"] +[19.113108, "o", "v"] +[19.163194, "o", "e"] +[19.213318, "o", "n"] +[19.33145, "o", "a"] +[19.381515, "o", "n"] +[19.437635, "o", "c"] +[19.487692, "o", "e\r\n\u001b[?2004l\r"] +[19.487835, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[20.488206, "o", "c"] +[20.562293, "o", "u"] +[20.612377, "o", "r"] +[20.678526, "o", "l"] +[20.728611, "o", " "] +[20.830744, "o", "-"] +[20.903826, "o", "s"] +[21.012945, "o", "L"] +[21.072046, "o", "O"] +[21.122141, "o", " "] +[21.172243, "o", "h"] +[21.222366, "o", "t"] +[21.293417, "o", "t"] +[21.34358, "o", "p"] +[21.540715, "o", "s"] +[21.590749, "o", ":"] +[21.656855, "o", "/"] +[21.706987, "o", "/"] +[21.92115, "o", "g"] +[21.971204, "o", "i"] +[22.021325, "o", "t"] +[22.071434, "o", "h"] +[22.142549, "o", "u"] +[22.192631, "o", "b"] +[22.242749, "o", "."] +[22.292845, "o", "c"] +[22.407959, "o", "o"] +[22.458051, "o", "m"] +[22.508157, "o", "/"] +[22.55825, "o", "e"] +[22.608382, "o", "d"] +[22.658485, "o", "g"] +[22.708595, "o", "e"] +[22.758753, "o", "l"] +[22.808856, "o", "e"] +[22.858937, "o", "s"] +[22.909048, "o", "s"] +[22.959138, "o", "s"] +[23.009238, "o", "y"] +[23.090365, "o", "s"] +[23.140466, "o", "/"] +[23.224602, "o", "c"] +[23.311677, "o", "o"] +[23.361771, "o", "n"] +[23.543871, "o", "s"] +[23.625969, "o", "t"] +[23.696098, "o", "e"] +[23.746145, "o", "l"] +[23.796313, "o", "l"] +[23.846396, "o", "a"] +[23.896517, "o", "t"] +[23.946604, "o", "i"] +[24.045732, "o", "o"] +[24.095818, "o", "n"] +[24.145929, "o", "/"] +[24.19609, "o", "r"] +[24.246194, "o", "e"] +[24.296292, "o", "l"] +[24.346388, "o", "e"] +[24.396503, "o", "a"] +[24.446571, "o", "s"] +[24.496712, "o", "e"] +[24.546813, "o", "s"] +[24.59694, "o", "/"] +[24.647003, "o", "l"] +[24.703116, "o", "a"] +[24.816297, "o", "t"] +[24.866358, "o", "e"] +[24.93547, "o", "s"] +[24.985531, "o", "t"] +[25.035649, "o", "/"] +[25.085721, "o", "d"] +[25.199862, "o", "o"] +[25.256966, "o", "w"] +[25.307067, "o", "n"] +[25.358187, "o", "l"] +[25.408287, "o", "o"] +[25.536394, "o", "a"] +[25.646532, "o", "d"] +[25.69662, "o", "/"] +[25.746734, "o", "c"] +[25.796816, "o", "o"] +[25.846951, "o", "n"] +[25.897059, "o", "s"] +[26.068166, "o", "t"] +[26.14026, "o", "e"] +[26.190353, "o", "l"] +[26.240519, "o", "l"] +[26.356613, "o", "a"] +[26.472726, "o", "t"] +[26.522826, "o", "i"] +[26.573028, "o", "o"] +[26.623055, "o", "n"] +[26.673156, "o", "-"] +[26.741276, "o", "l"] +[26.791381, "o", "i"] +[26.841508, "o", "n"] +[26.892594, "o", "u"] +[26.942668, "o", "x"] +[26.992815, "o", "-"] +[27.05791, "o", "a"] +[27.108074, "o", "m"] +[27.158082, "o", "d"] +[27.208208, "o", "6"] +[27.258355, "o", "4\r\n\u001b[?2004l\r"] +[27.947176, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[27.947448, "o", "c"] +[28.030593, "o", "u"] +[28.081746, "o", "r"] +[28.131861, "o", "l"] +[28.18195, "o", " "] +[28.255058, "o", "-"] +[28.318205, "o", "s"] +[28.36827, "o", "L"] +[28.418377, "o", "O"] +[28.468478, "o", " "] +[28.518512, "o", "h"] +[28.610681, "o", "t"] +[28.660794, "o", "t"] +[28.717914, "o", "p"] +[28.786017, "o", "s"] +[28.836116, "o", ":"] +[28.886224, "o", "/"] +[28.936285, "o", "/"] +[28.986504, "o", "g"] +[29.041612, "o", "i"] +[29.144711, "o", "t"] +[29.194811, "o", "h"] +[29.317912, "o", "u"] +[29.367988, "o", "b"] +[29.418088, "o", "."] +[29.539231, "o", "c"] +[29.616322, "o", "o"] +[29.666503, "o", "m"] +[29.716544, "o", "/"] +[29.766695, "o", "e"] +[29.816774, "o", "d"] +[29.866885, "o", "g"] +[29.940003, "o", "e"] +[29.994097, "o", "l"] +[30.062214, "o", "e"] +[30.112293, "o", "s"] +[30.162413, "o", "s"] +[30.212508, "o", "s"] +[30.315663, "o", "y"] +[30.365806, "o", "s"] +[30.415869, "o", "/"] +[30.465976, "o", "c"] +[30.516083, "o", "o"] +[30.566192, "o", "n"] +[30.700334, "o", "s"] +[30.75044, "o", "t"] +[30.800555, "o", "e"] +[30.882637, "o", "l"] +[31.10477, "o", "l"] +[31.156845, "o", "a"] +[31.206932, "o", "t"] +[31.25708, "o", "i"] +[31.342178, "o", "o"] +[31.392285, "o", "n"] +[31.442392, "o", "/"] +[31.492516, "o", "r"] +[31.542606, "o", "e"] +[31.592708, "o", "l"] +[31.720857, "o", "e"] +[31.770946, "o", "a"] +[31.825086, "o", "s"] +[31.965188, "o", "e"] +[32.017269, "o", "s"] +[32.067394, "o", "/"] +[32.117485, "o", "l"] +[32.167612, "o", "a"] +[32.222706, "o", "t"] +[32.303792, "o", "e"] +[32.35392, "o", "s"] +[32.404015, "o", "t"] +[32.454093, "o", "/"] +[32.504231, "o", "d"] +[32.58228, "o", "o"] +[32.632375, "o", "w"] +[32.682523, "o", "n"] +[32.746566, "o", "l"] +[32.796728, "o", "o"] +[32.849803, "o", "a"] +[32.89992, "o", "d"] +[32.949989, "o", "/"] +[33.000137, "o", "c"] +[33.076209, "o", "o"] +[33.133332, "o", "n"] +[33.24744, "o", "s"] +[33.400592, "o", "t"] +[33.450639, "o", "e"] +[33.500747, "o", "l"] +[33.550865, "o", "l"] +[33.645976, "o", "a"] +[33.753073, "o", "t"] +[33.803192, "o", "i"] +[33.853225, "o", "o"] +[33.939357, "o", "n"] +[33.989478, "o", "."] +[34.039587, "o", "i"] +[34.089697, "o", "n"] +[34.139777, "o", "t"] +[34.18988, "o", "o"] +[34.240003, "o", "t"] +[34.290032, "o", "o"] +[34.340137, "o", "."] +[34.390195, "o", "j"] +[34.440284, "o", "s"] +[34.490345, "o", "o"] +[34.558508, "o", "n"] +[34.608603, "o", "l\r\n\u001b[?2004l\r"] +[35.478619, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[36.479061, "o", "#"] +[36.529102, "o", " "] +[36.714251, "o", "S"] +[36.764324, "o", "t"] +[36.814432, "o", "e"] +[36.864554, "o", "p"] +[36.914634, "o", " "] +[36.989759, "o", "3"] +[37.039864, "o", ":"] +[37.089953, "o", " "] +[37.218025, "o", "V"] +[37.339179, "o", "e"] +[37.389254, "o", "r"] +[37.463359, "o", "i"] +[37.541502, "o", "f"] +[37.627597, "o", "y"] +[37.677701, "o", " "] +[37.727803, "o", "p"] +[37.777913, "o", "r"] +[37.828015, "o", "o"] +[37.911124, "o", "v"] +[37.961185, "o", "e"] +[38.011306, "o", "n"] +[38.093415, "o", "a"] +[38.150469, "o", "n"] +[38.213603, "o", "c"] +[38.263694, "o", "e\r\n\u001b[?2004l\r"] +[38.263829, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[39.264242, "o", "s"] +[39.334319, "o", "l"] +[39.384431, "o", "s"] +[39.434553, "o", "a"] +[39.484656, "o", "-"] +[39.540765, "o", "v"] +[39.590857, "o", "e"] +[39.640976, "o", "r"] +[39.691089, "o", "i"] +[39.84824, "o", "f"] +[40.019372, "o", "i"] +[40.069438, "o", "e"] +[40.125559, "o", "r"] +[40.175627, "o", " "] +[40.225789, "o", "v"] +[40.275891, "o", "e"] +[40.325998, "o", "r"] +[40.403112, "o", "i"] +[40.453193, "o", "f"] +[40.519301, "o", "y"] +[40.569404, "o", "-"] +[40.61951, "o", "a"] +[40.669603, "o", "r"] +[40.802716, "o", "t"] +[40.852817, "o", "i"] +[40.902919, "o", "f"] +[40.953025, "o", "a"] +[41.084147, "o", "c"] +[41.19826, "o", "t"] +[41.248342, "o", " "] +[41.333462, "o", "c"] +[41.400624, "o", "o"] +[41.45069, "o", "n"] +[41.500797, "o", "s"] +[41.614917, "o", "t"] +[41.665004, "o", "e"] +[41.778123, "o", "l"] +[41.858263, "o", "l"] +[41.908347, "o", "a"] +[41.958453, "o", "t"] +[42.028563, "o", "i"] +[42.078755, "o", "o"] +[42.12884, "o", "n"] +[42.178933, "o", "-"] +[42.229029, "o", "l"] +[42.279094, "o", "i"] +[42.329226, "o", "n"] +[42.379332, "o", "u"] +[42.429438, "o", "x"] +[42.479537, "o", "-"] +[42.529646, "o", "a"] +[42.590747, "o", "m"] +[42.669864, "o", "d"] +[42.719955, "o", "6"] +[42.770061, "o", "4"] +[42.820156, "o", " "] +[42.870327, "o", "-"] +[42.92045, "o", "-"] +[42.970528, "o", "p"] +[43.020648, "o", "r"] +[43.095734, "o", "o"] +[43.145848, "o", "v"] +[43.247957, "o", "e"] +[43.313099, "o", "n"] +[43.363188, "o", "a"] +[43.413265, "o", "n"] +[43.58544, "o", "c"] +[43.635504, "o", "e"] +[43.685607, "o", "-"] +[43.735736, "o", "p"] +[43.785821, "o", "a"] +[43.886963, "o", "t"] +[43.937046, "o", "h"] +[43.987146, "o", " "] +[44.037226, "o", "c"] +[44.121355, "o", "o"] +[44.17145, "o", "n"] +[44.221593, "o", "s"] +[44.271637, "o", "t"] +[44.321766, "o", "e"] +[44.376868, "o", "l"] +[44.426962, "o", "l"] +[44.560102, "o", "a"] +[44.610144, "o", "t"] +[44.660275, "o", "i"] +[44.71037, "o", "o"] +[44.760473, "o", "n"] +[44.810585, "o", "."] +[44.860706, "o", "i"] +[44.910767, "o", "n"] +[44.9609, "o", "t"] +[45.011013, "o", "o"] +[45.061109, "o", "t"] +[45.11519, "o", "o"] +[45.165304, "o", "."] +[45.296454, "o", "j"] +[45.346521, "o", "s"] +[45.407649, "o", "o"] +[45.457735, "o", "n"] +[45.507902, "o", "l"] +[45.557962, "o", " "] +[45.608053, "o", "-"] +[45.658162, "o", "-"] +[45.708273, "o", "s"] +[45.822397, "o", "o"] +[45.872465, "o", "u"] +[45.922582, "o", "r"] +[45.97269, "o", "c"] +[46.022794, "o", "e"] +[46.072893, "o", "-"] +[46.123013, "o", "u"] +[46.225142, "o", "r"] +[46.27524, "o", "i"] +[46.325343, "o", " "] +[46.375406, "o", "g"] +[46.44856, "o", "i"] +[46.498639, "o", "t"] +[46.548766, "o", "h"] +[46.598869, "o", "u"] +[46.648984, "o", "b"] +[46.699126, "o", "."] +[46.749173, "o", "c"] +[46.799292, "o", "o"] +[46.889399, "o", "m"] +[46.939484, "o", "/"] +[47.021624, "o", "e"] +[47.07172, "o", "d"] +[47.171837, "o", "g"] +[47.221925, "o", "e"] +[47.272033, "o", "l"] +[47.322127, "o", "e"] +[47.419249, "o", "s"] +[47.469339, "o", "s"] +[47.519452, "o", "s"] +[47.569547, "o", "y"] +[47.619666, "o", "s"] +[47.669786, "o", "/"] +[47.772993, "o", "c"] +[47.823, "o", "o"] +[47.873098, "o", "n"] +[47.954195, "o", "s"] +[48.0043, "o", "t"] +[48.05443, "o", "e"] +[48.104505, "o", "l"] +[48.154623, "o", "l"] +[48.258749, "o", "a"] +[48.31683, "o", "t"] +[48.366944, "o", "i"] +[48.483045, "o", "o"] +[48.533144, "o", "n\r\n\u001b[?2004l\r"] +[49.335338, "o", "Verified signature against tlog entry index 57885093 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77a78f50198d0b0a4554049eec73d59a7e5e1f895c4ad5b3d202c62132aea18c4c1\r\n"] +[49.349461, "o", "Verified build using builder \"https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.9.0\" at commit 1d05f438ffa564d2e4ac3b071dbaf322e6d954ab\r\nVerifying artifact constellation-linux-amd64: PASSED\r\n\r\nPASSED: Verified SLSA provenance\r\n"] +[49.350696, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[49.350872, "o", "#"] +[49.401022, "o", " "] +[49.451237, "o", "S"] +[49.501261, "o", "t"] +[49.591428, "o", "e"] +[49.641525, "o", "p"] +[49.691629, "o", " "] +[49.741693, "o", "4"] +[49.79182, "o", ":"] +[49.886917, "o", " "] +[49.937036, "o", "I"] +[50.025111, "o", "n"] +[50.08623, "o", "s"] +[50.182343, "o", "t"] +[50.343467, "o", "a"] +[50.393546, "o", "l"] +[50.51966, "o", "l"] +[50.569734, "o", " "] +[50.619894, "o", "t"] +[50.670013, "o", "h"] +[50.720095, "o", "e"] +[50.7702, "o", " "] +[50.820323, "o", "C"] +[50.870433, "o", "L"] +[50.995579, "o", "I\r\n\u001b[?2004l\r"] +[50.995627, "o", "\u001b[?2004h"] +[50.995743, "o", "\u001b[38;2;139;4;221m$\u001b[0m "] +[51.996149, "o", "s"] +[52.050207, "o", "u"] +[52.100327, "o", "d"] +[52.150387, "o", "o"] +[52.200522, "o", " "] +[52.250618, "o", "i"] +[52.300732, "o", "n"] +[52.380832, "o", "s"] +[52.430902, "o", "t"] +[52.481038, "o", "a"] +[52.531129, "o", "l"] +[52.581232, "o", "l"] +[52.631309, "o", " "] +[52.705492, "o", "c"] +[52.755569, "o", "o"] +[52.843666, "o", "n"] +[52.893795, "o", "s"] +[52.995901, "o", "t"] +[53.046011, "o", "e"] +[53.142094, "o", "l"] +[53.23122, "o", "l"] +[53.330348, "o", "a"] +[53.404454, "o", "t"] +[53.467566, "o", "i"] +[53.517633, "o", "o"] +[53.567768, "o", "n"] +[53.617875, "o", "-"] +[53.667986, "o", "l"] +[53.784085, "o", "i"] +[53.834192, "o", "n"] +[54.033356, "o", "u"] +[54.08346, "o", "x"] +[54.133557, "o", "-"] +[54.183666, "o", "a"] +[54.233756, "o", "m"] +[54.283884, "o", "d"] +[54.379993, "o", "6"] +[54.430107, "o", "4"] +[54.480217, "o", " "] +[54.53029, "o", "/"] +[54.580429, "o", "u"] +[54.630513, "o", "s"] +[54.680659, "o", "r"] +[54.730762, "o", "/"] +[54.780844, "o", "l"] +[54.836972, "o", "o"] +[54.924054, "o", "c"] +[54.974166, "o", "a"] +[55.024298, "o", "l"] +[55.074406, "o", "/"] +[55.18749, "o", "b"] +[55.315578, "o", "i"] +[55.615744, "o", "n"] +[55.665855, "o", "/"] +[55.716954, "o", "c"] +[55.767026, "o", "o"] +[56.050153, "o", "n"] +[56.100236, "o", "s"] +[56.21236, "o", "t"] +[56.267461, "o", "e"] +[56.317549, "o", "l"] +[56.368677, "o", "l"] +[56.434788, "o", "a"] +[56.484867, "o", "t"] +[56.535006, "o", "i"] +[56.625075, "o", "o"] +[56.753129, "o", "n\r\n\u001b[?2004l\r"] +[56.786535, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[57.786967, "o", "#"] +[57.837066, "o", " "] +[57.887144, "o", "D"] +[57.986273, "o", "o"] +[58.036354, "o", "n"] +[58.086476, "o", "e"] +[58.136575, "o", "!"] +[58.224681, "o", " "] +[58.274788, "o", "Y"] +[58.337896, "o", "o"] +[58.390972, "o", "u"] +[58.441103, "o", " "] +[58.491206, "o", "c"] +[58.541277, "o", "a"] +[58.591411, "o", "n"] +[58.641516, "o", " "] +[58.691628, "o", "n"] +[58.751733, "o", "o"] +[58.801851, "o", "w"] +[58.851969, "o", " "] +[58.911044, "o", "u"] +[58.961123, "o", "s"] +[59.011257, "o", "e"] +[59.061363, "o", " "] +[59.12652, "o", "t"] +[59.185589, "o", "h"] +[59.286705, "o", "e"] +[59.336784, "o", " "] +[59.386882, "o", "v"] +[59.446964, "o", "e"] +[59.497085, "o", "r"] +[59.596274, "o", "i"] +[59.646336, "o", "f"] +[59.737469, "o", "i"] +[59.787561, "o", "e"] +[59.981692, "o", "d"] +[60.031767, "o", " "] +[60.10389, "o", "C"] +[60.186941, "o", "L"] +[60.306082, "o", "I\r\n\u001b[?2004l\r"] +[60.306124, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] +[61.306557, "o", "c"] +[61.356611, "o", "o"] +[61.406688, "o", "n"] +[61.456814, "o", "s"] +[61.535927, "o", "t"] +[61.60803, "o", "e"] +[61.658174, "o", "l"] +[61.708252, "o", "l"] +[61.758334, "o", "a"] +[61.808441, "o", "t"] +[61.858536, "o", "i"] +[61.91863, "o", "o"] +[61.975698, "o", "n"] +[62.025846, "o", " "] +[62.115954, "o", "-"] +[62.16603, "o", "h\r\n\u001b[?2004l\r"] +[62.195248, "o", "Manage your Constellation cluster.\r\n\r\n"] +[62.195729, "o", "Usage:\r\n constellation [command]\r\n\r\nAvailable Commands:\r\n config Work with the Constellation configuration file\r\n apply Apply a configuration to a Constellation cluster\r\n mini Manage MiniConstellation clusters\r\n status Show status of a Constellation cluster\r\n verify Verify the confidential properties of a Constellation cluster\r\n upgrade Find and apply upgrades to your Constellation cluster\r\n recover Recover a completely stopped Constellation cluster\r\n terminate Terminate a Constellation cluster\r\n iam Work with the IAM configuration on your cloud provider\r\n version Display version of this CLI\r\n help Help about any command\r\n completion Generate the autocompletion script for the specified shell\r\n\r\nFlags:\r\n --debug enable debug logging\r\n --force disable version compatibility checks - might result in corrupted clusters\r\n -h, --help help for constellation\r\n --tf-log string Terraform lo"] +[62.195843, "o", "g level (default \"NONE\")\r\n -C, --workspace string path to the Constellation workspace\r\n\r\nUse \"constellation [command] --help\" for more information about a command.\r\n"] +[62.197435, "o", "\u001b[?2004h\u001b[38;2;139;4;221m$\u001b[0m "] diff --git a/docs/static/gtagman.js b/docs/static/gtagman.js new file mode 100644 index 000000000..57bf6717a --- /dev/null +++ b/docs/static/gtagman.js @@ -0,0 +1,5 @@ +(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': +new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], +j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= +'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); +})(window,document,'script','dataLayer','GTM-NF9NM7V'); diff --git a/docs/static/img/BannerConstellationanimated.svg b/docs/static/img/BannerConstellationanimated.svg new file mode 100644 index 000000000..f937f69a0 --- /dev/null +++ b/docs/static/img/BannerConstellationanimated.svg @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/static/img/banner.svg b/docs/static/img/banner.svg deleted file mode 100644 index cb19bc6e9..000000000 --- a/docs/static/img/banner.svg +++ /dev/null @@ -1,155 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/static/img/concept.svg b/docs/static/img/concept.svg index cfb9a0039..2286ff308 100644 --- a/docs/static/img/concept.svg +++ b/docs/static/img/concept.svg @@ -1,1021 +1,201 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/static/img/shell-windowframe.svg b/docs/static/img/shell-windowframe.svg index bb547c209..65d844638 100644 --- a/docs/static/img/shell-windowframe.svg +++ b/docs/static/img/shell-windowframe.svg @@ -16,138 +16,291 @@ } :root { - --animation-duration: 11116ms; + --animation-duration: 21558ms; } @keyframes roll { 0.000%{transform:translateY(0px)} -0.054%{transform:translateY(-306px)} -1.250%{transform:translateY(-612px)} -1.700%{transform:translateY(-918px)} -2.150%{transform:translateY(-1224px)} -2.789%{transform:translateY(-1530px)} -3.239%{transform:translateY(-1836px)} -3.688%{transform:translateY(-2142px)} -4.768%{transform:translateY(-2448px)} -5.227%{transform:translateY(-2754px)} -5.677%{transform:translateY(-3060px)} -6.126%{transform:translateY(-3366px)} -7.089%{transform:translateY(-3672px)} -7.539%{transform:translateY(-3978px)} -7.988%{transform:translateY(-4284px)} -8.438%{transform:translateY(-4590px)} -8.888%{transform:translateY(-4896px)} -10.408%{transform:translateY(-5202px)} -11.083%{transform:translateY(-5508px)} -11.884%{transform:translateY(-5814px)} -12.334%{transform:translateY(-6120px)} -16.832%{transform:translateY(-6426px)} -21.330%{transform:translateY(-6732px)} -21.402%{transform:translateY(-7038px)} -21.851%{transform:translateY(-7344px)} -22.400%{transform:translateY(-7650px)} -23.354%{transform:translateY(-7956px)} -23.804%{transform:translateY(-8262px)} -24.262%{transform:translateY(-8568px)} -24.712%{transform:translateY(-8874px)} -25.162%{transform:translateY(-9180px)} -25.612%{transform:translateY(-9486px)} -26.062%{transform:translateY(-9792px)} -26.970%{transform:translateY(-10098px)} -27.420%{transform:translateY(-10404px)} -27.870%{transform:translateY(-10710px)} -28.320%{transform:translateY(-11016px)} -28.778%{transform:translateY(-11322px)} -29.228%{transform:translateY(-11628px)} -29.678%{transform:translateY(-11934px)} -30.901%{transform:translateY(-12240px)} -31.414%{transform:translateY(-12546px)} -31.972%{transform:translateY(-12852px)} -36.470%{transform:translateY(-13158px)} -40.968%{transform:translateY(-13464px)} -45.466%{transform:translateY(-13770px)} -45.565%{transform:translateY(-14076px)} -46.024%{transform:translateY(-14382px)} -46.617%{transform:translateY(-14688px)} -47.067%{transform:translateY(-14994px)} -47.850%{transform:translateY(-15300px)} -48.300%{transform:translateY(-15606px)} -48.750%{transform:translateY(-15912px)} -49.280%{transform:translateY(-16218px)} -49.730%{transform:translateY(-16524px)} -50.684%{transform:translateY(-16830px)} -51.134%{transform:translateY(-17136px)} -51.583%{transform:translateY(-17442px)} -52.303%{transform:translateY(-17748px)} -53.212%{transform:translateY(-18054px)} -53.661%{transform:translateY(-18360px)} -54.165%{transform:translateY(-18666px)} -55.227%{transform:translateY(-18972px)} -55.964%{transform:translateY(-19278px)} -56.414%{transform:translateY(-19584px)} -56.864%{transform:translateY(-19890px)} -57.926%{transform:translateY(-20196px)} -58.375%{transform:translateY(-20502px)} -59.482%{transform:translateY(-20808px)} -59.932%{transform:translateY(-21114px)} -62.100%{transform:translateY(-21420px)} -63.053%{transform:translateY(-21726px)} -63.575%{transform:translateY(-22032px)} -64.223%{transform:translateY(-22338px)} -64.673%{transform:translateY(-22644px)} -65.131%{transform:translateY(-22950px)} -65.581%{transform:translateY(-23256px)} -66.328%{transform:translateY(-23562px)} -66.778%{transform:translateY(-23868px)} -68.199%{transform:translateY(-24174px)} -68.649%{transform:translateY(-24480px)} -69.108%{transform:translateY(-24786px)} -69.557%{transform:translateY(-25092px)} -70.007%{transform:translateY(-25398px)} -70.502%{transform:translateY(-25704px)} -70.952%{transform:translateY(-26010px)} -71.402%{transform:translateY(-26316px)} -72.841%{transform:translateY(-26622px)} -73.291%{transform:translateY(-26928px)} -74.973%{transform:translateY(-27234px)} -75.765%{transform:translateY(-27540px)} -76.214%{transform:translateY(-27846px)} -76.664%{transform:translateY(-28152px)} -77.114%{transform:translateY(-28458px)} -77.564%{transform:translateY(-28764px)} -78.014%{transform:translateY(-29070px)} -78.463%{transform:translateY(-29376px)} -78.922%{transform:translateY(-29682px)} -79.372%{transform:translateY(-29988px)} -79.822%{transform:translateY(-30294px)} -80.272%{transform:translateY(-30600px)} -80.766%{transform:translateY(-30906px)} -81.216%{transform:translateY(-31212px)} -85.714%{transform:translateY(-31518px)} -86.164%{transform:translateY(-31824px)} -86.614%{transform:translateY(-32130px)} -87.073%{transform:translateY(-32436px)} -87.549%{transform:translateY(-32742px)} -88.224%{transform:translateY(-33048px)} -88.692%{transform:translateY(-33354px)} -89.142%{transform:translateY(-33660px)} -89.745%{transform:translateY(-33966px)} -90.194%{transform:translateY(-34272px)} -90.671%{transform:translateY(-34578px)} -91.130%{transform:translateY(-34884px)} -91.580%{transform:translateY(-35190px)} -92.200%{transform:translateY(-35496px)} -92.650%{transform:translateY(-35802px)} -93.370%{transform:translateY(-36108px)} -93.820%{transform:translateY(-36414px)} -95.466%{transform:translateY(-36720px)} -95.493%{transform:translateY(-37026px)} -99.991%{transform:translateY(-37332px)} -100.000%{transform:translateY(-37332px)} +0.019%{transform:translateY(-306px)} +0.631%{transform:translateY(-612px)} +0.863%{transform:translateY(-918px)} +1.095%{transform:translateY(-1224px)} +1.424%{transform:translateY(-1530px)} +1.656%{transform:translateY(-1836px)} +1.888%{transform:translateY(-2142px)} +2.449%{transform:translateY(-2448px)} +2.681%{transform:translateY(-2754px)} +2.913%{transform:translateY(-3060px)} +3.145%{transform:translateY(-3366px)} +3.641%{transform:translateY(-3672px)} +3.873%{transform:translateY(-3978px)} +4.105%{transform:translateY(-4284px)} +4.337%{transform:translateY(-4590px)} +4.569%{transform:translateY(-4896px)} +5.348%{transform:translateY(-5202px)} +5.701%{transform:translateY(-5508px)} +6.114%{transform:translateY(-5814px)} +6.346%{transform:translateY(-6120px)} +6.578%{transform:translateY(-6426px)} +6.810%{transform:translateY(-6732px)} +7.092%{transform:translateY(-7038px)} +7.584%{transform:translateY(-7344px)} +7.816%{transform:translateY(-7650px)} +8.048%{transform:translateY(-7956px)} +8.280%{transform:translateY(-8262px)} +8.512%{transform:translateY(-8568px)} +8.748%{transform:translateY(-8874px)} +8.980%{transform:translateY(-9180px)} +9.449%{transform:translateY(-9486px)} +9.681%{transform:translateY(-9792px)} +9.913%{transform:translateY(-10098px)} +10.038%{transform:translateY(-10404px)} +10.047%{transform:translateY(-10710px)} +12.367%{transform:translateY(-11016px)} +12.599%{transform:translateY(-11322px)} +12.831%{transform:translateY(-11628px)} +13.062%{transform:translateY(-11934px)} +13.693%{transform:translateY(-12240px)} +13.958%{transform:translateY(-12546px)} +14.259%{transform:translateY(-12852px)} +14.491%{transform:translateY(-13158px)} +14.797%{transform:translateY(-13464px)} +15.034%{transform:translateY(-13770px)} +15.437%{transform:translateY(-14076px)} +15.669%{transform:translateY(-14382px)} +15.901%{transform:translateY(-14688px)} +16.133%{transform:translateY(-14994px)} +16.365%{transform:translateY(-15300px)} +16.857%{transform:translateY(-15606px)} +17.089%{transform:translateY(-15912px)} +17.321%{transform:translateY(-16218px)} +17.692%{transform:translateY(-16524px)} +18.165%{transform:translateY(-16830px)} +18.397%{transform:translateY(-17136px)} +18.657%{transform:translateY(-17442px)} +19.204%{transform:translateY(-17748px)} +19.584%{transform:translateY(-18054px)} +19.816%{transform:translateY(-18360px)} +20.048%{transform:translateY(-18666px)} +20.591%{transform:translateY(-18972px)} +20.823%{transform:translateY(-19278px)} +21.059%{transform:translateY(-19584px)} +21.291%{transform:translateY(-19890px)} +22.409%{transform:translateY(-20196px)} +22.901%{transform:translateY(-20502px)} +23.170%{transform:translateY(-20808px)} +23.504%{transform:translateY(-21114px)} +23.736%{transform:translateY(-21420px)} +23.968%{transform:translateY(-21726px)} +24.200%{transform:translateY(-22032px)} +24.432%{transform:translateY(-22338px)} +24.668%{transform:translateY(-22644px)} +25.401%{transform:translateY(-22950px)} +25.633%{transform:translateY(-23256px)} +25.865%{transform:translateY(-23562px)} +26.097%{transform:translateY(-23868px)} +26.329%{transform:translateY(-24174px)} +26.561%{transform:translateY(-24480px)} +26.793%{transform:translateY(-24786px)} +27.025%{transform:translateY(-25092px)} +27.772%{transform:translateY(-25398px)} +28.004%{transform:translateY(-25704px)} +28.871%{transform:translateY(-26010px)} +29.275%{transform:translateY(-26316px)} +29.506%{transform:translateY(-26622px)} +29.738%{transform:translateY(-26928px)} +29.970%{transform:translateY(-27234px)} +30.202%{transform:translateY(-27540px)} +30.434%{transform:translateY(-27846px)} +30.671%{transform:translateY(-28152px)} +30.903%{transform:translateY(-28458px)} +31.464%{transform:translateY(-28764px)} +31.696%{transform:translateY(-29070px)} +31.928%{transform:translateY(-29376px)} +32.183%{transform:translateY(-29682px)} +32.415%{transform:translateY(-29988px)} +32.883%{transform:translateY(-30294px)} +33.115%{transform:translateY(-30600px)} +33.347%{transform:translateY(-30906px)} +33.584%{transform:translateY(-31212px)} +33.830%{transform:translateY(-31518px)} +34.178%{transform:translateY(-31824px)} +34.419%{transform:translateY(-32130px)} +36.460%{transform:translateY(-32436px)} +36.831%{transform:translateY(-32742px)} +37.063%{transform:translateY(-33048px)} +37.810%{transform:translateY(-33354px)} +38.042%{transform:translateY(-33660px)} +38.273%{transform:translateY(-33966px)} +38.505%{transform:translateY(-34272px)} +39.039%{transform:translateY(-34578px)} +39.271%{transform:translateY(-34884px)} +39.619%{transform:translateY(-35190px)} +39.851%{transform:translateY(-35496px)} +40.083%{transform:translateY(-35802px)} +40.315%{transform:translateY(-36108px)} +40.546%{transform:translateY(-36414px)} +40.834%{transform:translateY(-36720px)} +41.066%{transform:translateY(-37026px)} +41.298%{transform:translateY(-37332px)} +41.534%{transform:translateY(-37638px)} +41.766%{transform:translateY(-37944px)} +41.998%{transform:translateY(-38250px)} +42.230%{transform:translateY(-38556px)} +42.574%{transform:translateY(-38862px)} +42.805%{transform:translateY(-39168px)} +43.181%{transform:translateY(-39474px)} +43.413%{transform:translateY(-39780px)} +43.803%{transform:translateY(-40086px)} +44.039%{transform:translateY(-40392px)} +44.434%{transform:translateY(-40698px)} +44.666%{transform:translateY(-41004px)} +44.897%{transform:translateY(-41310px)} +45.129%{transform:translateY(-41616px)} +45.547%{transform:translateY(-41922px)} +45.969%{transform:translateY(-42228px)} +46.201%{transform:translateY(-42534px)} +46.433%{transform:translateY(-42840px)} +46.669%{transform:translateY(-43146px)} +47.152%{transform:translateY(-43452px)} +47.843%{transform:translateY(-43758px)} +48.075%{transform:translateY(-44064px)} +48.307%{transform:translateY(-44370px)} +48.539%{transform:translateY(-44676px)} +48.771%{transform:translateY(-44982px)} +49.003%{transform:translateY(-45288px)} +49.304%{transform:translateY(-45594px)} +49.541%{transform:translateY(-45900px)} +49.773%{transform:translateY(-46206px)} +50.005%{transform:translateY(-46512px)} +50.260%{transform:translateY(-46818px)} +50.492%{transform:translateY(-47124px)} +51.396%{transform:translateY(-47430px)} +52.194%{transform:translateY(-47736px)} +52.426%{transform:translateY(-48042px)} +52.658%{transform:translateY(-48348px)} +53.048%{transform:translateY(-48654px)} +53.280%{transform:translateY(-48960px)} +53.544%{transform:translateY(-49266px)} +53.776%{transform:translateY(-49572px)} +54.008%{transform:translateY(-49878px)} +54.240%{transform:translateY(-50184px)} +54.472%{transform:translateY(-50490px)} +54.704%{transform:translateY(-50796px)} +54.936%{transform:translateY(-51102px)} +55.167%{transform:translateY(-51408px)} +55.311%{transform:translateY(-51714px)} +57.631%{transform:translateY(-52020px)} +57.635%{transform:translateY(-52326px)} +57.644%{transform:translateY(-52632px)} +57.876%{transform:translateY(-52938px)} +58.108%{transform:translateY(-53244px)} +58.340%{transform:translateY(-53550px)} +58.670%{transform:translateY(-53856px)} +58.902%{transform:translateY(-54162px)} +59.134%{transform:translateY(-54468px)} +59.500%{transform:translateY(-54774px)} +59.732%{transform:translateY(-55080px)} +60.019%{transform:translateY(-55386px)} +60.251%{transform:translateY(-55692px)} +60.483%{transform:translateY(-55998px)} +60.715%{transform:translateY(-56304px)} +60.947%{transform:translateY(-56610px)} +61.290%{transform:translateY(-56916px)} +61.587%{transform:translateY(-57222px)} +61.819%{transform:translateY(-57528px)} +62.056%{transform:translateY(-57834px)} +62.325%{transform:translateY(-58140px)} +62.557%{transform:translateY(-58446px)} +62.789%{transform:translateY(-58752px)} +63.021%{transform:translateY(-59058px)} +63.164%{transform:translateY(-59364px)} +65.484%{transform:translateY(-59670px)} +67.803%{transform:translateY(-59976px)} +70.122%{transform:translateY(-60282px)} +70.141%{transform:translateY(-60588px)} +72.460%{transform:translateY(-60894px)} +72.474%{transform:translateY(-61200px)} +72.776%{transform:translateY(-61506px)} +73.309%{transform:translateY(-61812px)} +74.014%{transform:translateY(-62118px)} +75.406%{transform:translateY(-62424px)} +75.986%{transform:translateY(-62730px)} +76.222%{transform:translateY(-63036px)} +76.454%{transform:translateY(-63342px)} +76.686%{transform:translateY(-63648px)} +76.918%{transform:translateY(-63954px)} +77.150%{transform:translateY(-64260px)} +77.382%{transform:translateY(-64566px)} +77.614%{transform:translateY(-64872px)} +77.846%{transform:translateY(-65178px)} +78.078%{transform:translateY(-65484px)} +78.310%{transform:translateY(-65790px)} +78.704%{transform:translateY(-66096px)} +79.084%{transform:translateY(-66402px)} +79.316%{transform:translateY(-66708px)} +79.994%{transform:translateY(-67014px)} +80.286%{transform:translateY(-67320px)} +80.903%{transform:translateY(-67626px)} +81.162%{transform:translateY(-67932px)} +81.394%{transform:translateY(-68238px)} +81.640%{transform:translateY(-68544px)} +81.877%{transform:translateY(-68850px)} +82.234%{transform:translateY(-69156px)} +82.466%{transform:translateY(-69462px)} +82.698%{transform:translateY(-69768px)} +82.930%{transform:translateY(-70074px)} +83.162%{transform:translateY(-70380px)} +83.394%{transform:translateY(-70686px)} +83.626%{transform:translateY(-70992px)} +83.858%{transform:translateY(-71298px)} +84.094%{transform:translateY(-71604px)} +84.326%{transform:translateY(-71910px)} +84.641%{transform:translateY(-72216px)} +84.873%{transform:translateY(-72522px)} +85.105%{transform:translateY(-72828px)} +85.337%{transform:translateY(-73134px)} +85.569%{transform:translateY(-73440px)} +85.880%{transform:translateY(-73746px)} +86.112%{transform:translateY(-74052px)} +86.344%{transform:translateY(-74358px)} +86.678%{transform:translateY(-74664px)} +87.411%{transform:translateY(-74970px)} +87.643%{transform:translateY(-75276px)} +87.875%{transform:translateY(-75582px)} +88.107%{transform:translateY(-75888px)} +88.357%{transform:translateY(-76194px)} +88.589%{transform:translateY(-76500px)} +88.821%{transform:translateY(-76806px)} +89.053%{transform:translateY(-77112px)} +89.846%{transform:translateY(-77418px)} +90.078%{transform:translateY(-77724px)} +90.310%{transform:translateY(-78030px)} +90.542%{transform:translateY(-78336px)} +92.861%{transform:translateY(-78642px)} +93.408%{transform:translateY(-78948px)} +93.640%{transform:translateY(-79254px)} +93.900%{transform:translateY(-79560px)} +94.132%{transform:translateY(-79866px)} +94.364%{transform:translateY(-80172px)} +94.707%{transform:translateY(-80478px)} +94.944%{transform:translateY(-80784px)} +95.250%{transform:translateY(-81090px)} +95.487%{transform:translateY(-81396px)} +95.960%{transform:translateY(-81702px)} +96.192%{transform:translateY(-82008px)} +96.697%{transform:translateY(-82314px)} +96.971%{transform:translateY(-82620px)} +97.203%{transform:translateY(-82926px)} +97.435%{transform:translateY(-83232px)} +97.671%{transform:translateY(-83538px)} +99.991%{transform:translateY(-83844px)} +99.995%{transform:translateY(-84150px)} +100.000%{transform:translateY(-84150px)} } #screen_view { - animation-duration: 11116ms; + animation-duration: 21558ms; animation-iteration-count:infinite; animation-name:roll; animation-timing-function: steps(1,end); @@ -183,5 +336,5 @@ - $ c $ co $ con $ cons $ const $ conste $ constel $ constell $ constella $ constellat $ constellati $ constellatio $ constellation $ constellation $ constellation c $ constellation cr $ constellation cre $ constellation crea $ constellation creat $ constellation createThe following Constellation cluster will be created: 3 control-plane nodes of type n2d-standard-4 will be created. 1 worker node of type n2d-standard-4 will be created.Do you want to create this cluster? [y/n]: yYour Constellation cluster was created successfully.$ constellation a $ constellation ap $ constellation app $ constellation appl $ constellation applyUsing community license.For details, see https://docs.edgeless.systems/constellation/overview/licenseYour Constellation master secret was successfully written to "constellation-mastersecret.json"Your Constellation cluster was successfully initialized.Constellation cluster identifier 9e6e04e154bdd280dc0e8810eccf132fb15fd443216a3d983347c004a64177a5Kubernetes configuration constellation-admin.confYou can now connect to your cluster by executing:export KUBECONFIG="/constellation/constellation-admin.conf"$ e $ ex $ exp $ expo $ expor $ export $ export $ export K $ export KU $ export KUB $ export KUBE $ export KUBEC $ export KUBECO $ export KUBECON $ export KUBECONF $ export KUBECONFI $ export KUBECONFIG $ export KUBECONFIG= $ export KUBECONFIG=/ $ export KUBECONFIG=/c $ export KUBECONFIG=/co $ export KUBECONFIG=/con $ export KUBECONFIG=/cons $ export KUBECONFIG=/const $ export KUBECONFIG=/conste $ export KUBECONFIG=/constel $ export KUBECONFIG=/constell $ export KUBECONFIG=/constella $ export KUBECONFIG=/constellat $ export KUBECONFIG=/constellati $ export KUBECONFIG=/constellatio $ export KUBECONFIG=/constellation $ export KUBECONFIG=/constellation/ $ export KUBECONFIG=/constellation/c $ export KUBECONFIG=/constellation/co $ export KUBECONFIG=/constellation/con $ export KUBECONFIG=/constellation/cons $ export KUBECONFIG=/constellation/const $ export KUBECONFIG=/constellation/conste $ export KUBECONFIG=/constellation/constel $ export KUBECONFIG=/constellation/constell $ export KUBECONFIG=/constellation/constella $ export KUBECONFIG=/constellation/constellat $ export KUBECONFIG=/constellation/constellati $ export KUBECONFIG=/constellation/constellatio $ export KUBECONFIG=/constellation/constellation $ export KUBECONFIG=/constellation/constellation- $ export KUBECONFIG=/constellation/constellation-a $ export KUBECONFIG=/constellation/constellation-ad $ export KUBECONFIG=/constellation/constellation-adm $ export KUBECONFIG=/constellation/constellation-admi $ export KUBECONFIG=/constellation/constellation-admin $ export KUBECONFIG=/constellation/constellation-admin. $ export KUBECONFIG=/constellation/constellation-admin.c $ export KUBECONFIG=/constellation/constellation-admin.co $ export KUBECONFIG=/constellation/constellation-admin.con $ export KUBECONFIG=/constellation/constellation-admin.conf$ $ k $ ku $ kub $ kube $ kubec $ kubect $ kubectl $ kubectl $ kubectl g $ kubectl ge $ kubectl get $ kubectl get $ kubectl get n $ kubectl get no $ kubectl get nod $ kubectl get node $ kubectl get nodesNAME STATUS ROLES AGE VERSIONconstell-5013563c-control-plane-45468ae4-4s9d Ready control-plane 2m37s v1.27.7$ $ exit + $ c $ co $ con $ cons $ const $ conste $ constel $ constell $ constella $ constellat $ constellati $ constellatio $ constellation $ constellation $ constellation c $ constellation co $ constellation con $ constellation conf $ constellation confi $ constellation config $ constellation config $ constellation config g $ constellation config ge $ constellation config gen $ constellation config gene $ constellation config gener $ constellation config genera $ constellation config generat $ constellation config generate $ constellation config generate $ constellation config generate g $ constellation config generate gc $ constellation config generate gcpConfig file written to constellation-conf.yamlPlease fill in your CSP-specific configuration before proceeding.State file written to constellation-state.yamlFor more information refer to the documentation:https://docs.edgeless.systems/constellation/getting-started/first-steps$ $ constellation i $ constellation ia $ constellation iam $ constellation iam $ constellation iam c $ constellation iam cr $ constellation iam cre $ constellation iam crea $ constellation iam creat $ constellation iam create $ constellation iam create $ constellation iam create g $ constellation iam create gc $ constellation iam create gcp $ constellation iam create gcp $ constellation iam create gcp - $ constellation iam create gcp -- $ constellation iam create gcp --u $ constellation iam create gcp --up $ constellation iam create gcp --upd $ constellation iam create gcp --upda $ constellation iam create gcp --updat $ constellation iam create gcp --update $ constellation iam create gcp --update- $ constellation iam create gcp --update-c $ constellation iam create gcp --update-co $ constellation iam create gcp --update-con $ constellation iam create gcp --update-conf $ constellation iam create gcp --update-confi $ constellation iam create gcp --update-config $ constellation iam create gcp --update-config $ constellation iam create gcp --update-config - $ constellation iam create gcp --update-config -- $ constellation iam create gcp --update-config --p $ constellation iam create gcp --update-config --pr $ constellation iam create gcp --update-config --pro $ constellation iam create gcp --update-config --proj $ constellation iam create gcp --update-config --proje $ constellation iam create gcp --update-config --projec $ constellation iam create gcp --update-config --project $ constellation iam create gcp --update-config --projectI $ constellation iam create gcp --update-config --projectID $ constellation iam create gcp --update-config --projectID $ constellation iam create gcp --update-config --projectID c $ constellation iam create gcp --update-config --projectID co $ constellation iam create gcp --update-config --projectID con $ constellation iam create gcp --update-config --projectID cons $ constellation iam create gcp --update-config --projectID const $ constellation iam create gcp --update-config --projectID conste $ constellation iam create gcp --update-config --projectID constel $ constellation iam create gcp --update-config --projectID constell $ constellation iam create gcp --update-config --projectID constella $ constellation iam create gcp --update-config --projectID constellat $ constellation iam create gcp --update-config --projectID constellati $ constellation iam create gcp --update-config --projectID constellatio $ constellation iam create gcp --update-config --projectID constellation $ constellation iam create gcp --update-config --projectID constellation $ constellation iam create gcp --update-config --projectID constellation - $ constellation iam create gcp --update-config --projectID constellation -- $ constellation iam create gcp --update-config --projectID constellation --s $ constellation iam create gcp --update-config --projectID constellation --se $ constellation iam create gcp --update-config --projectID constellation --ser $ constellation iam create gcp --update-config --projectID constellation --serv $ constellation iam create gcp --update-config --projectID constellation --servi $ constellation iam create gcp --update-config --projectID constellation --servic $ constellation iam create gcp --update-config --projectID constellation --service $ constellation iam create gcp --update-config --projectID constellation --serviceA $ constellation iam create gcp --update-config --projectID constellation --serviceAc $ constellation iam create gcp --update-config --projectID constellation --serviceAcc $ constellation iam create gcp --update-config --projectID constellation --serviceAcco $ constellation iam create gcp --update-config --projectID constellation --serviceAccou $ constellation iam create gcp --update-config --projectID constellation --serviceAccoun $ constellation iam create gcp --update-config --projectID constellation --serviceAccount $ constellation iam create gcp --update-config --projectID constellation --serviceAccountI $ constellation iam create gcp --update-config --projectID constellation --serviceAccountID $ constellation iam create gcp --update-config --projectID constellation --serviceAccountID $ constellation iam create gcp --update-config --projectID constellation --serviceAccountID c $ constellation iam create gcp --update-config --projectID constellation --serviceAccountID co $ constellation iam create gcp --update-config --projectID constellation --serviceAccountID con $ constellation iam create gcp --update-config --projectID constellation --serviceAccountID cons st ste stel stell stella stellat stellati stellatio stellation stellation- stellation-d stellation-de stellation-dem stellation-demo stellation-demo stellation-demo - stellation-demo -- stellation-demo --z stellation-demo --zo stellation-demo --zon stellation-demo --zone stellation-demo --zone stellation-demo --zone e stellation-demo --zone eu stellation-demo --zone eur stellation-demo --zone euro stellation-demo --zone europ stellation-demo --zone europe stellation-demo --zone europe- stellation-demo --zone europe-w stellation-demo --zone europe-we stellation-demo --zone europe-wes stellation-demo --zone europe-west stellation-demo --zone europe-west3 stellation-demo --zone europe-west3- stellation-demo --zone europe-west3-b stellation-demo --zone europe-west3-b stellation-demo --zone europe-west3-b - stellation-demo --zone europe-west3-b -yThe configuration file "constellation-conf.yaml" will be automatically updated with the IAM values and zone/region information.Your IAM configuration was created and filled into constellation-conf.yaml successfully.$ constellation a $ constellation ap $ constellation app $ constellation appl $ constellation apply $ constellation apply $ constellation apply - $ constellation apply -yUsing community license.For details, see https://docs.edgeless.systems/constellation/overview/licenseThe following Constellation cluster will be created: 3 control-plane nodes of type n2d-standard-4 will be created. 1 worker node of type n2d-standard-4 will be created.Cloud infrastructure created successfully.Your Constellation master secret was successfully written to "constellation-mastersecret.json"Your Constellation cluster was successfully initialized.Constellation cluster identifier 9a33e294b6f63d3593cdc85e0c4cba55f0e04800e2c9dd846e17da68c16364b9Kubernetes configuration constellation-admin.confYou can now connect to your cluster by executing:export KUBECONFIG="/constellation/constellation-admin.conf"$ e $ ex $ exp $ expo $ expor $ export $ export $ export K $ export KU $ export KUB $ export KUBE $ export KUBEC $ export KUBECO $ export KUBECON $ export KUBECONF $ export KUBECONFI $ export KUBECONFIG $ export KUBECONFIG= $ export KUBECONFIG=/ $ export KUBECONFIG=/c $ export KUBECONFIG=/co $ export KUBECONFIG=/con $ export KUBECONFIG=/cons $ export KUBECONFIG=/const $ export KUBECONFIG=/conste $ export KUBECONFIG=/constel $ export KUBECONFIG=/constell $ export KUBECONFIG=/constella $ export KUBECONFIG=/constellat $ export KUBECONFIG=/constellati $ export KUBECONFIG=/constellatio $ export KUBECONFIG=/constellation $ export KUBECONFIG=/constellation/ $ export KUBECONFIG=/constellation/c $ export KUBECONFIG=/constellation/co $ export KUBECONFIG=/constellation/con $ export KUBECONFIG=/constellation/cons $ export KUBECONFIG=/constellation/const $ export KUBECONFIG=/constellation/conste $ export KUBECONFIG=/constellation/constel $ export KUBECONFIG=/constellation/constell $ export KUBECONFIG=/constellation/constella $ export KUBECONFIG=/constellation/constellat $ export KUBECONFIG=/constellation/constellati $ export KUBECONFIG=/constellation/constellatio $ export KUBECONFIG=/constellation/constellation $ export KUBECONFIG=/constellation/constellation- $ export KUBECONFIG=/constellation/constellation-a $ export KUBECONFIG=/constellation/constellation-ad $ export KUBECONFIG=/constellation/constellation-adm $ export KUBECONFIG=/constellation/constellation-admi $ export KUBECONFIG=/constellation/constellation-admin $ export KUBECONFIG=/constellation/constellation-admin. $ export KUBECONFIG=/constellation/constellation-admin.c $ export KUBECONFIG=/constellation/constellation-admin.co $ export KUBECONFIG=/constellation/constellation-admin.con $ export KUBECONFIG=/constellation/constellation-admin.conf$ k $ ku $ kub $ kube $ kubec $ kubect $ kubectl $ kubectl $ kubectl g $ kubectl ge $ kubectl get $ kubectl get $ kubectl get n $ kubectl get no $ kubectl get nod $ kubectl get node $ kubectl get nodesNAME STATUS ROLES AGE VERSIONconstell-aad70a14-control-plane-65328c70-6kfn Ready control-plane 3m27s v1.27.8$ \ No newline at end of file diff --git a/docs/styles/Google/OptionalPlurals.yml b/docs/styles/Google/OptionalPlurals.yml deleted file mode 100644 index f858ea6fe..000000000 --- a/docs/styles/Google/OptionalPlurals.yml +++ /dev/null @@ -1,12 +0,0 @@ -extends: existence -message: "Don't use plurals in parentheses such as in '%s'." -link: 'https://developers.google.com/style/plurals-parentheses' -level: error -nonword: true -action: - name: edit - params: - - remove - - '(s)' -tokens: - - '\b\w+\(s\)' diff --git a/docs/styles/Google/Spelling.yml b/docs/styles/Google/Spelling.yml index 57acb8841..527ac07d3 100644 --- a/docs/styles/Google/Spelling.yml +++ b/docs/styles/Google/Spelling.yml @@ -5,4 +5,6 @@ ignorecase: true level: warning tokens: - '(?:\w+)nised?' - - '(?:\w+)logue' + - 'colour' + - 'labour' + - 'centre' diff --git a/docs/styles/Google/Units.yml b/docs/styles/Google/Units.yml index 379fad6b8..53522ab2d 100644 --- a/docs/styles/Google/Units.yml +++ b/docs/styles/Google/Units.yml @@ -1,8 +1,8 @@ extends: existence message: "Put a nonbreaking space between the number and the unit in '%s'." -link: 'https://developers.google.com/style/units-of-measure' +link: "https://developers.google.com/style/units-of-measure" nonword: true level: error tokens: - - \d+(?:B|kB|MB|GB|TB) - - \d+(?:ns|ms|s|min|h|d) + - \b\d+(?:B|kB|MB|GB|TB) + - \b\d+(?:ns|ms|s|min|h|d) diff --git a/docs/styles/Microsoft/Contractions.yml b/docs/styles/Microsoft/Contractions.yml index ded330f79..8c81dcbce 100644 --- a/docs/styles/Microsoft/Contractions.yml +++ b/docs/styles/Microsoft/Contractions.yml @@ -22,7 +22,7 @@ swap: should not: shouldn't - 'that is(?!\.)': that's + "that is(?![.,])": that's 'that''s(?=\.)': that is 'they are(?!\.)': they're diff --git a/docs/styles/Microsoft/Foreign.yml b/docs/styles/Microsoft/Foreign.yml index d37835a5d..0d3d6002a 100644 --- a/docs/styles/Microsoft/Foreign.yml +++ b/docs/styles/Microsoft/Foreign.yml @@ -9,4 +9,5 @@ action: swap: '\b(?:eg|e\.g\.)[\s,]': for example '\b(?:ie|i\.e\.)[\s,]': that is - + '\b(?:viz\.)[\s,]': namely + '\b(?:ergo)[\s,]': therefore diff --git a/docs/styles/Microsoft/Plurals.yml b/docs/styles/Microsoft/Plurals.yml new file mode 100644 index 000000000..1bb6660ad --- /dev/null +++ b/docs/styles/Microsoft/Plurals.yml @@ -0,0 +1,7 @@ +extends: existence +message: "Don't add '%s' to a singular noun. Use plural instead." +ignorecase: true +level: error +link: https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/s/s-es +raw: + - '\(s\)|\(es\)' diff --git a/docs/styles/Microsoft/Terms.yml b/docs/styles/Microsoft/Terms.yml index e41ff74b7..7708900eb 100644 --- a/docs/styles/Microsoft/Terms.yml +++ b/docs/styles/Microsoft/Terms.yml @@ -1,5 +1,7 @@ extends: substitution message: "Prefer '%s' over '%s'." +# term preference should be based on microsoft style guide, such as +link: https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/adapter level: warning ignorecase: true action: @@ -24,10 +26,8 @@ swap: anti-spyware: antispyware anti-virus: antivirus appendixes: appendices - artificial intelligence: artificial intelligence - assembler: assembly language - bpp: bpp - bps: bps + artificial intelligence: AI + '(?:assembler|machine language)': assembly language caap: CaaP conversation-as-a-platform: conversation as a platform eb: EB @@ -39,5 +39,3 @@ swap: pb: PB tb: TB zb: ZB - viz: namely - ergo: therefore diff --git a/docs/styles/Vocab/constellation/accept.txt b/docs/styles/config/vocabularies/edgeless/accept.txt similarity index 69% rename from docs/styles/Vocab/constellation/accept.txt rename to docs/styles/config/vocabularies/edgeless/accept.txt index 59cb8c22c..1676e071f 100644 --- a/docs/styles/Vocab/constellation/accept.txt +++ b/docs/styles/config/vocabularies/edgeless/accept.txt @@ -2,69 +2,84 @@ Affero agent Ansys Asciinema +ASG auditable autoscaler -[Aa]utoscaling +autoscaling AWS -aws backend Bazel -benchmarked -[Bb]ootloader +bootloader Bootstrapper -[Cc]loud +CLI +cloud config +CPU +CRD +CSP cyber datacenter Datadog deallocate +DEK Dockerfile Dynatrace -[Ee]mojivoto +Emojivoto etcd Filebeat Filestash Filestore +filesystem Fluentd +frontend Fulcio -Mbps Gbps GCP -gcp Grype -iam IAM -iodepth initramfs +iodepth +IPSec journald -[Kk]3s Kata +KEK +KMS kubeadm kubectl kubelet libcryptsetup Logstash +Mbps MicroK8s -[Mm]inikube namespace Nginx -[Pp]laintext +paravisor +PCR +plaintext +proxied +QEMU Rekor resizable +resourceGroup rollout +SBOM sigstore -[Ss]uperset +SSD +STACKIT +superset Syft systemd +Terraform Tink -[Uu]nencrypted +TPM +unencrypted unspoofable updatable UUID -proxied -QEMU +vCPU virsh -[Ww]alkthrough +VM +walkthrough whitepaper WireGuard Xeon diff --git a/docs/versioned_docs/version-2.0/architecture/attestation.md b/docs/versioned_docs/version-2.0/architecture/attestation.md index 443c19639..92ee2e9a2 100644 --- a/docs/versioned_docs/version-2.0/architecture/attestation.md +++ b/docs/versioned_docs/version-2.0/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -149,8 +149,8 @@ The latter means that the value can be generated offline and compared to the one | 12 | ClusterID | Constellation Bootstrapper | Yes | | 13–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -179,8 +179,8 @@ The latter means that the value can be generated offline and compared to the one | 12 | ClusterID | Constellation Bootstrapper | Yes | | 13–23 | Unused |- | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.0/architecture/keys.md b/docs/versioned_docs/version-2.0/architecture/keys.md index cb8c41768..ae6044862 100644 --- a/docs/versioned_docs/version-2.0/architecture/keys.md +++ b/docs/versioned_docs/version-2.0/architecture/keys.md @@ -101,7 +101,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.0/getting-started/first-steps.md b/docs/versioned_docs/version-2.0/getting-started/first-steps.md index 9be306396..caec42ded 100644 --- a/docs/versioned_docs/version-2.0/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.0/getting-started/first-steps.md @@ -6,29 +6,29 @@ The following steps guide you through the process of creating a cluster and depl 1. Create the configuration file for your selected cloud provider. - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + This creates the file `constellation-conf.yaml` in your current working directory. 2. Fill in your cloud provider specific information. - - + + You need several resources for the cluster. You can use the following `az` script to create them: @@ -59,8 +59,8 @@ The following steps guide you through the process of creating a cluster and depl Run `constellation config instance-types` to get the list of all supported options. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -106,8 +106,8 @@ The following steps guide you through the process of creating a cluster and depl Run `constellation config instance-types` to get the list of all supported options. - - + + You need a service account for the cluster. You can use the following `gcloud` script to create it: @@ -130,18 +130,18 @@ The following steps guide you through the process of creating a cluster and depl By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines from the N2D family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). - * **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. + * **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). - * **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. + * **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -159,8 +159,8 @@ The following steps guide you through the process of creating a cluster and depl Supported are all machines from the N2D family with a minimum of 4 vCPUs. It defaults to `n2d-standard-4` (4 vCPUs, 16 GB RAM), but you can use any other VMs from the same family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + :::info diff --git a/docs/versioned_docs/version-2.0/getting-started/install.md b/docs/versioned_docs/version-2.0/getting-started/install.md index 8e3545c1a..5945f0405 100644 --- a/docs/versioned_docs/version-2.0/getting-started/install.md +++ b/docs/versioned_docs/version-2.0/getting-started/install.md @@ -18,8 +18,8 @@ Make sure the following requirements are met: The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -104,8 +101,8 @@ The CLI makes authenticated calls to the CSP API. Therefore, you need to set up ### Required permissions - - + + You need the following permissions for your user account: @@ -115,8 +112,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -128,8 +125,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + ### Authentication @@ -139,8 +136,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -156,8 +153,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -180,8 +177,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + ## Next steps diff --git a/docs/versioned_docs/version-2.0/overview/clouds.md b/docs/versioned_docs/version-2.0/overview/clouds.md index c526d956b..0fa4b79f7 100644 --- a/docs/versioned_docs/version-2.0/overview/clouds.md +++ b/docs/versioned_docs/version-2.0/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.0/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.0/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.0/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.0/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.0/workflows/create.md b/docs/versioned_docs/version-2.0/workflows/create.md index 357ab6703..a426202e5 100644 --- a/docs/versioned_docs/version-2.0/workflows/create.md +++ b/docs/versioned_docs/version-2.0/workflows/create.md @@ -15,22 +15,22 @@ This step creates the necessary resources for your cluster in your cloud environ Generate a configuration file for your cloud service provider (CSP): - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. [Fill in your CSP-specific information](../getting-started/first-steps.md#create-a-cluster) before you continue. @@ -53,7 +53,7 @@ constellation create --control-plane-nodes 1 --worker-nodes 2 For details on the flags, consult the command help via `constellation create -h`. -*create* stores your cluster's configuration to a file named [`constellation-state.json`](../architecture/orchestration.md#installation-process) in your current directory. +*create* stores your cluster's configuration to a file named [`constellation-state.json`](../architecture/orchestration.md#cluster-creation-process) in your current directory. ## The *init* step diff --git a/docs/versioned_docs/version-2.0/workflows/recovery.md b/docs/versioned_docs/version-2.0/workflows/recovery.md index 4c6010d98..ca37ca839 100644 --- a/docs/versioned_docs/version-2.0/workflows/recovery.md +++ b/docs/versioned_docs/version-2.0/workflows/recovery.md @@ -17,8 +17,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -52,8 +52,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. For this, you need its IP address, which you can obtain from the *Overview* page under *Private IP address*. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -88,8 +88,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. For this, you need its IP address, which you can obtain from the **VM Instance** > **network interfaces** table under *Primary internal IP address*. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.0/workflows/scale.md b/docs/versioned_docs/version-2.0/workflows/scale.md index 3318d8aee..84773b80c 100644 --- a/docs/versioned_docs/version-2.0/workflows/scale.md +++ b/docs/versioned_docs/version-2.0/workflows/scale.md @@ -6,23 +6,23 @@ Constellation provides all features of a Kubernetes cluster including scaling an [During cluster initialization](create.md#the-init-step) you can choose to deploy the [cluster autoscaler](https://github.com/kubernetes/autoscaler). It automatically provisions additional worker nodes so that all pods have a place to run. Alternatively, you can choose to manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + ## Control-plane node scaling @@ -30,23 +30,23 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.0/workflows/storage.md b/docs/versioned_docs/version-2.0/workflows/storage.md index 958c73261..38d77c694 100644 --- a/docs/versioned_docs/version-2.0/workflows/storage.md +++ b/docs/versioned_docs/version-2.0/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -45,8 +45,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The following installation guide gives an overview of how to securely use CSI-based cloud storage for persistent volumes in Constellation. - - + + 1. Install the driver: @@ -56,8 +56,8 @@ The following installation guide gives an overview of how to securely use CSI-ba helm install azuredisk-csi-driver charts/edgeless --namespace kube-system ``` - - + + 1. Install the driver: @@ -66,8 +66,8 @@ The following installation guide gives an overview of how to securely use CSI-ba helm install gcp-compute-persistent-disk-csi-driver charts/ --namespace kube-system ``` - - + + :::info @@ -138,8 +138,8 @@ The default storage class is responsible for all persistent volume claims that d The previous instructions create a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -185,8 +185,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -232,5 +232,5 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + diff --git a/docs/versioned_docs/version-2.0/workflows/troubleshooting.md b/docs/versioned_docs/version-2.0/workflows/troubleshooting.md index ba340601b..afc9274c6 100644 --- a/docs/versioned_docs/version-2.0/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.0/workflows/troubleshooting.md @@ -8,8 +8,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -19,8 +19,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -35,5 +35,5 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + diff --git a/docs/versioned_docs/version-2.0/workflows/verify-cli.md b/docs/versioned_docs/version-2.0/workflows/verify-cli.md index 0a52fedd4..52ed24d95 100644 --- a/docs/versioned_docs/version-2.0/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.0/workflows/verify-cli.md @@ -1,6 +1,6 @@ # Verify the CLI -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -12,7 +12,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -25,7 +25,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.1/architecture/attestation.md b/docs/versioned_docs/version-2.1/architecture/attestation.md index 443c19639..92ee2e9a2 100644 --- a/docs/versioned_docs/version-2.1/architecture/attestation.md +++ b/docs/versioned_docs/version-2.1/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -149,8 +149,8 @@ The latter means that the value can be generated offline and compared to the one | 12 | ClusterID | Constellation Bootstrapper | Yes | | 13–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -179,8 +179,8 @@ The latter means that the value can be generated offline and compared to the one | 12 | ClusterID | Constellation Bootstrapper | Yes | | 13–23 | Unused |- | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.1/architecture/keys.md b/docs/versioned_docs/version-2.1/architecture/keys.md index cb8c41768..ae6044862 100644 --- a/docs/versioned_docs/version-2.1/architecture/keys.md +++ b/docs/versioned_docs/version-2.1/architecture/keys.md @@ -101,7 +101,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.1/getting-started/first-steps.md b/docs/versioned_docs/version-2.1/getting-started/first-steps.md index bd9513650..10cf3d7b9 100644 --- a/docs/versioned_docs/version-2.1/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.1/getting-started/first-steps.md @@ -11,29 +11,29 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step 1. Create the configuration file for your selected cloud provider. - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + This creates the file `constellation-conf.yaml` in your current working directory. 2. Fill in your cloud provider specific information. - - + + You need several resources for the cluster. You can use the following `az` script to create them: @@ -64,8 +64,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Run `constellation config instance-types` to get the list of all supported options. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -111,8 +111,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Run `constellation config instance-types` to get the list of all supported options. - - + + You need a service account for the cluster. You can use the following `gcloud` script to create it: @@ -135,18 +135,18 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines from the N2D family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). - * **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. + * **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). - * **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. + * **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -164,8 +164,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Supported are all machines from the N2D family with a minimum of 4 vCPUs. It defaults to `n2d-standard-4` (4 vCPUs, 16 GB RAM), but you can use any other VMs from the same family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + :::info diff --git a/docs/versioned_docs/version-2.1/getting-started/install.md b/docs/versioned_docs/version-2.1/getting-started/install.md index 56029b806..d4cf81ff7 100644 --- a/docs/versioned_docs/version-2.1/getting-started/install.md +++ b/docs/versioned_docs/version-2.1/getting-started/install.md @@ -18,8 +18,8 @@ Make sure the following requirements are met: The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,8 +105,8 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + You need the following permissions for your user account: @@ -119,8 +116,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -132,8 +129,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + ### Authentication @@ -143,8 +140,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -160,8 +157,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -184,8 +181,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + ## Next steps diff --git a/docs/versioned_docs/version-2.1/overview/clouds.md b/docs/versioned_docs/version-2.1/overview/clouds.md index c526d956b..0fa4b79f7 100644 --- a/docs/versioned_docs/version-2.1/overview/clouds.md +++ b/docs/versioned_docs/version-2.1/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.1/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.1/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.1/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.1/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.1/workflows/create.md b/docs/versioned_docs/version-2.1/workflows/create.md index a567c5d23..a8956d7e4 100644 --- a/docs/versioned_docs/version-2.1/workflows/create.md +++ b/docs/versioned_docs/version-2.1/workflows/create.md @@ -19,22 +19,22 @@ This step creates the necessary resources for your cluster in your cloud environ Generate a configuration file for your cloud service provider (CSP): - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. [Fill in your CSP-specific information](../getting-started/first-steps.md#create-a-cluster) before you continue. @@ -57,7 +57,7 @@ constellation create --control-plane-nodes 1 --worker-nodes 2 For details on the flags, consult the command help via `constellation create -h`. -*create* stores your cluster's configuration to a file named [`constellation-state.json`](../architecture/orchestration.md#installation-process) in your current directory. +*create* stores your cluster's configuration to a file named [`constellation-state.json`](../architecture/orchestration.md#cluster-creation-process) in your current directory. ## The *init* step diff --git a/docs/versioned_docs/version-2.1/workflows/recovery.md b/docs/versioned_docs/version-2.1/workflows/recovery.md index cde039ea7..c55daf413 100644 --- a/docs/versioned_docs/version-2.1/workflows/recovery.md +++ b/docs/versioned_docs/version-2.1/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.1/workflows/scale.md b/docs/versioned_docs/version-2.1/workflows/scale.md index 1c8757fe8..669856655 100644 --- a/docs/versioned_docs/version-2.1/workflows/scale.md +++ b/docs/versioned_docs/version-2.1/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + ## Control-plane node scaling @@ -72,23 +72,23 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.1/workflows/storage.md b/docs/versioned_docs/version-2.1/workflows/storage.md index 958c73261..38d77c694 100644 --- a/docs/versioned_docs/version-2.1/workflows/storage.md +++ b/docs/versioned_docs/version-2.1/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -45,8 +45,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The following installation guide gives an overview of how to securely use CSI-based cloud storage for persistent volumes in Constellation. - - + + 1. Install the driver: @@ -56,8 +56,8 @@ The following installation guide gives an overview of how to securely use CSI-ba helm install azuredisk-csi-driver charts/edgeless --namespace kube-system ``` - - + + 1. Install the driver: @@ -66,8 +66,8 @@ The following installation guide gives an overview of how to securely use CSI-ba helm install gcp-compute-persistent-disk-csi-driver charts/ --namespace kube-system ``` - - + + :::info @@ -138,8 +138,8 @@ The default storage class is responsible for all persistent volume claims that d The previous instructions create a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -185,8 +185,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -232,5 +232,5 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + diff --git a/docs/versioned_docs/version-2.1/workflows/troubleshooting.md b/docs/versioned_docs/version-2.1/workflows/troubleshooting.md index ba340601b..afc9274c6 100644 --- a/docs/versioned_docs/version-2.1/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.1/workflows/troubleshooting.md @@ -8,8 +8,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -19,8 +19,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -35,5 +35,5 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + diff --git a/docs/versioned_docs/version-2.1/workflows/verify-cli.md b/docs/versioned_docs/version-2.1/workflows/verify-cli.md index 0a52fedd4..52ed24d95 100644 --- a/docs/versioned_docs/version-2.1/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.1/workflows/verify-cli.md @@ -1,6 +1,6 @@ # Verify the CLI -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -12,7 +12,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -25,7 +25,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.10/architecture/attestation.md b/docs/versioned_docs/version-2.10/architecture/attestation.md index 07ac3aa72..592063193 100644 --- a/docs/versioned_docs/version-2.10/architecture/attestation.md +++ b/docs/versioned_docs/version-2.10/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,16 +217,16 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,18 +248,18 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + There is no additional configuration available for GCP. - - + + There is no additional configuration available for AWS. - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.10/architecture/keys.md b/docs/versioned_docs/version-2.10/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.10/architecture/keys.md +++ b/docs/versioned_docs/version-2.10/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.10/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.10/getting-started/first-steps-local.md index de9c66e9b..a6e825906 100644 --- a/docs/versioned_docs/version-2.10/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.10/getting-started/first-steps-local.md @@ -45,8 +45,8 @@ sudo iptables -P FORWARD ACCEPT ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -74,8 +74,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -151,8 +151,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -205,8 +205,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -217,8 +217,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -246,8 +246,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.10/getting-started/first-steps.md b/docs/versioned_docs/version-2.10/getting-started/first-steps.md index 0b224e04d..4420eb708 100644 --- a/docs/versioned_docs/version-2.10/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.10/getting-started/first-steps.md @@ -15,39 +15,39 @@ If you encounter any problem with the following steps, make sure to use the [lat 1. Create the [configuration file](../workflows/config.md) for your cloud provider. - + - + ```bash constellation config generate azure ``` - + - + ```bash constellation config generate gcp ``` - + - + ```bash constellation config generate aws ``` - + - + 2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config @@ -62,21 +62,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `westeurope` * `southeastasia` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --update-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config @@ -103,8 +103,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.10/getting-started/install.md b/docs/versioned_docs/version-2.10/getting-started/install.md index 4debbca9a..c21ad259c 100644 --- a/docs/versioned_docs/version-2.10/getting-started/install.md +++ b/docs/versioned_docs/version-2.10/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,39 +105,42 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` -* `Microsoft.Network/natGateways/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` +- `Microsoft.Network/natGateways/*` The built-in `Contributor` role is a superset of these permissions. @@ -148,91 +148,91 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -278,13 +278,12 @@ The built-in `AdministratorAccess` policy is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). - The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -294,8 +293,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -311,8 +310,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -335,8 +334,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -352,10 +351,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.10/overview/clouds.md b/docs/versioned_docs/version-2.10/overview/clouds.md index 3ccbb0d6d..dfc3d5307 100644 --- a/docs/versioned_docs/version-2.10/overview/clouds.md +++ b/docs/versioned_docs/version-2.10/overview/clouds.md @@ -31,11 +31,11 @@ This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. diff --git a/docs/versioned_docs/version-2.10/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.10/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.10/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.10/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.10/overview/performance/compute.md b/docs/versioned_docs/version-2.10/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.10/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.10/overview/performance/io.md b/docs/versioned_docs/version-2.10/overview/performance/io.md index dc7cf3d8b..3ae796f8a 100644 --- a/docs/versioned_docs/version-2.10/overview/performance/io.md +++ b/docs/versioned_docs/version-2.10/overview/performance/io.md @@ -58,7 +58,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. diff --git a/docs/versioned_docs/version-2.10/overview/performance/performance.md b/docs/versioned_docs/version-2.10/overview/performance/performance.md index 7f22a693e..59bf86602 100644 --- a/docs/versioned_docs/version-2.10/overview/performance/performance.md +++ b/docs/versioned_docs/version-2.10/overview/performance/performance.md @@ -1,18 +1,10 @@ # Performance analysis of Constellation -This section provides a comprehensive examination of the performance characteristics of Constellation, encompassing various aspects, including runtime encryption, I/O benchmarks, and real-world applications. +This section provides a comprehensive examination of the performance characteristics of Constellation. -## Impact of runtime encryption on performance +## Runtime encryption -All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. - -### AMD and Azure benchmarking - -AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. - -### AMD and Google benchmarking - -Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. ## I/O performance benchmarks diff --git a/docs/versioned_docs/version-2.10/overview/product.md b/docs/versioned_docs/version-2.10/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.10/overview/product.md +++ b/docs/versioned_docs/version-2.10/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.10/workflows/config.md b/docs/versioned_docs/version-2.10/workflows/config.md index 95f95aeec..edc3c9091 100644 --- a/docs/versioned_docs/version-2.10/workflows/config.md +++ b/docs/versioned_docs/version-2.10/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,49 +14,49 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. @@ -75,8 +75,8 @@ AWS is currently investigating the issue. SNP-based attestation will be enabled as soon as a fix is verified. ::: - - + + Fill the desired VM type into the **instanceType** fields in the `constellation-conf.yml` file. @@ -86,7 +86,6 @@ By default, Constellation creates the node groups `control_plane_default` and `w If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. Each node group can be scaled individually. - Consider the following example for AWS: ```yaml @@ -120,9 +119,9 @@ You can use the field `zone` to specify what availability zone nodes of the grou On Azure, this field is empty by default and nodes are automatically spread across availability zones. Consult the documentation of your cloud provider for more information: -- [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) -- [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) -- [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) ## Choosing a Kubernetes version @@ -134,8 +133,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -155,23 +154,23 @@ Note that CVMs are currently only supported in a few regions, check [Azure's pro Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -195,16 +194,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -233,19 +232,19 @@ The following describes the configuration fields and how you obtain the required The user-assigned identity is used by instances of the cluster to access other cloud resources. For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -259,9 +258,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -292,9 +291,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.10/workflows/create.md b/docs/versioned_docs/version-2.10/workflows/create.md index c0e0cd23d..dd56bc8b7 100644 --- a/docs/versioned_docs/version-2.10/workflows/create.md +++ b/docs/versioned_docs/version-2.10/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + ```bash constellation create @@ -35,8 +35,8 @@ constellation create *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -75,8 +75,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step diff --git a/docs/versioned_docs/version-2.10/workflows/recovery.md b/docs/versioned_docs/version-2.10/workflows/recovery.md index c26fb32eb..35596b8c9 100644 --- a/docs/versioned_docs/version-2.10/workflows/recovery.md +++ b/docs/versioned_docs/version-2.10/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.10/workflows/sbom.md b/docs/versioned_docs/version-2.10/workflows/sbom.md index 9ef6eb65c..6c1702dee 100644 --- a/docs/versioned_docs/version-2.10/workflows/sbom.md +++ b/docs/versioned_docs/version-2.10/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -11,13 +11,15 @@ SBOMs for Constellation are generated using [Syft](https://github.com/anchore/sy :::note The public key for Edgeless Systems' long-term code-signing key is: + ``` -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at https://edgeless.systems/es.pub and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -38,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.10/workflows/scale.md b/docs/versioned_docs/version-2.10/workflows/scale.md index 06898ad0c..63b727c7d 100644 --- a/docs/versioned_docs/version-2.10/workflows/scale.md +++ b/docs/versioned_docs/version-2.10/workflows/scale.md @@ -51,30 +51,30 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the worker ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + ## Control-plane node scaling @@ -82,30 +82,30 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.10/workflows/storage.md b/docs/versioned_docs/version-2.10/workflows/storage.md index 9e3d96346..06fbc4de6 100644 --- a/docs/versioned_docs/version-2.10/workflows/storage.md +++ b/docs/versioned_docs/version-2.10/workflows/storage.md @@ -21,30 +21,30 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. - - + + **Constellation CSI driver for AWS Elastic Block Store** Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -53,8 +53,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -82,8 +82,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -111,8 +111,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + AWS comes with two storage classes by default. @@ -140,8 +140,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/docs/versioned_docs/version-2.10/workflows/terminate.md b/docs/versioned_docs/version-2.10/workflows/terminate.md index 647eadb42..f33489ca5 100644 --- a/docs/versioned_docs/version-2.10/workflows/terminate.md +++ b/docs/versioned_docs/version-2.10/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.10/workflows/troubleshooting.md b/docs/versioned_docs/version-2.10/workflows/troubleshooting.md index 781cae8c5..38c0d87e9 100644 --- a/docs/versioned_docs/version-2.10/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.10/workflows/troubleshooting.md @@ -55,14 +55,12 @@ When in doubt, check if the encountered [issue is known](https://github.com/edge ::: - :::tip During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config-backup` config map in the `kube-system` namespace. To restore the old attestation config after a failed upgrade, you can copy the attestation config from this resource, put it in your configuration file and retry the upgrade. ::: - You can use the `upgrade apply` command to change measurements of a running cluster: 1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. @@ -84,8 +82,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -95,8 +93,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -111,16 +109,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.10/workflows/trusted-launch.md b/docs/versioned_docs/version-2.10/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.10/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.10/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.10/workflows/verify-cli.md b/docs/versioned_docs/version-2.10/workflows/verify-cli.md index 1280c51b0..e33569d37 100644 --- a/docs/versioned_docs/version-2.10/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.10/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,11 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.11/architecture/attestation.md b/docs/versioned_docs/version-2.11/architecture/attestation.md index 07ac3aa72..592063193 100644 --- a/docs/versioned_docs/version-2.11/architecture/attestation.md +++ b/docs/versioned_docs/version-2.11/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,16 +217,16 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,18 +248,18 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + There is no additional configuration available for GCP. - - + + There is no additional configuration available for AWS. - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.11/architecture/keys.md b/docs/versioned_docs/version-2.11/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.11/architecture/keys.md +++ b/docs/versioned_docs/version-2.11/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.11/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.11/getting-started/first-steps-local.md index de9c66e9b..a6e825906 100644 --- a/docs/versioned_docs/version-2.11/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.11/getting-started/first-steps-local.md @@ -45,8 +45,8 @@ sudo iptables -P FORWARD ACCEPT ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -74,8 +74,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -151,8 +151,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -205,8 +205,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -217,8 +217,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -246,8 +246,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.11/getting-started/first-steps.md b/docs/versioned_docs/version-2.11/getting-started/first-steps.md index 07b7f8410..9ebe21701 100644 --- a/docs/versioned_docs/version-2.11/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.11/getting-started/first-steps.md @@ -15,39 +15,39 @@ If you encounter any problem with the following steps, make sure to use the [lat 1. Create the [configuration file](../workflows/config.md) for your cloud provider. - + - + ```bash constellation config generate azure ``` - + - + ```bash constellation config generate gcp ``` - + - + ```bash constellation config generate aws ``` - + - + 2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config @@ -62,21 +62,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `westeurope` * `southeastasia` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --update-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config @@ -103,8 +103,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.11/getting-started/install.md b/docs/versioned_docs/version-2.11/getting-started/install.md index 4debbca9a..f134ca3c2 100644 --- a/docs/versioned_docs/version-2.11/getting-started/install.md +++ b/docs/versioned_docs/version-2.11/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,39 +105,42 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` \[2] +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` -* `Microsoft.Network/natGateways/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` +- `Microsoft.Network/natGateways/*` The built-in `Contributor` role is a superset of these permissions. @@ -148,91 +148,91 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -278,13 +278,12 @@ The built-in `AdministratorAccess` policy is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). - The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -294,8 +293,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -311,8 +310,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -335,8 +334,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -352,10 +351,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.11/overview/clouds.md b/docs/versioned_docs/version-2.11/overview/clouds.md index 3ccbb0d6d..dfc3d5307 100644 --- a/docs/versioned_docs/version-2.11/overview/clouds.md +++ b/docs/versioned_docs/version-2.11/overview/clouds.md @@ -31,11 +31,11 @@ This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. diff --git a/docs/versioned_docs/version-2.11/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.11/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.11/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.11/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.11/overview/performance/compute.md b/docs/versioned_docs/version-2.11/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.11/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.11/overview/performance/io.md b/docs/versioned_docs/version-2.11/overview/performance/io.md index dc7cf3d8b..3ae796f8a 100644 --- a/docs/versioned_docs/version-2.11/overview/performance/io.md +++ b/docs/versioned_docs/version-2.11/overview/performance/io.md @@ -58,7 +58,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. diff --git a/docs/versioned_docs/version-2.11/overview/performance/performance.md b/docs/versioned_docs/version-2.11/overview/performance/performance.md index 7f22a693e..59bf86602 100644 --- a/docs/versioned_docs/version-2.11/overview/performance/performance.md +++ b/docs/versioned_docs/version-2.11/overview/performance/performance.md @@ -1,18 +1,10 @@ # Performance analysis of Constellation -This section provides a comprehensive examination of the performance characteristics of Constellation, encompassing various aspects, including runtime encryption, I/O benchmarks, and real-world applications. +This section provides a comprehensive examination of the performance characteristics of Constellation. -## Impact of runtime encryption on performance +## Runtime encryption -All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. - -### AMD and Azure benchmarking - -AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. - -### AMD and Google benchmarking - -Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. ## I/O performance benchmarks diff --git a/docs/versioned_docs/version-2.11/overview/product.md b/docs/versioned_docs/version-2.11/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.11/overview/product.md +++ b/docs/versioned_docs/version-2.11/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.11/workflows/config.md b/docs/versioned_docs/version-2.11/workflows/config.md index 95f95aeec..edc3c9091 100644 --- a/docs/versioned_docs/version-2.11/workflows/config.md +++ b/docs/versioned_docs/version-2.11/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,49 +14,49 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. @@ -75,8 +75,8 @@ AWS is currently investigating the issue. SNP-based attestation will be enabled as soon as a fix is verified. ::: - - + + Fill the desired VM type into the **instanceType** fields in the `constellation-conf.yml` file. @@ -86,7 +86,6 @@ By default, Constellation creates the node groups `control_plane_default` and `w If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. Each node group can be scaled individually. - Consider the following example for AWS: ```yaml @@ -120,9 +119,9 @@ You can use the field `zone` to specify what availability zone nodes of the grou On Azure, this field is empty by default and nodes are automatically spread across availability zones. Consult the documentation of your cloud provider for more information: -- [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) -- [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) -- [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) ## Choosing a Kubernetes version @@ -134,8 +133,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -155,23 +154,23 @@ Note that CVMs are currently only supported in a few regions, check [Azure's pro Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -195,16 +194,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -233,19 +232,19 @@ The following describes the configuration fields and how you obtain the required The user-assigned identity is used by instances of the cluster to access other cloud resources. For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -259,9 +258,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -292,9 +291,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.11/workflows/create.md b/docs/versioned_docs/version-2.11/workflows/create.md index d2b0adf90..8dd4946de 100644 --- a/docs/versioned_docs/version-2.11/workflows/create.md +++ b/docs/versioned_docs/version-2.11/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + ```bash constellation create @@ -35,8 +35,8 @@ constellation create *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -75,8 +75,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step @@ -94,6 +94,6 @@ export KUBECONFIG="$PWD/constellation-admin.conf" 🏁 That's it. You've successfully created a Constellation cluster. - ### Troubleshooting + In case `init` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.11/workflows/recovery.md b/docs/versioned_docs/version-2.11/workflows/recovery.md index c26fb32eb..35596b8c9 100644 --- a/docs/versioned_docs/version-2.11/workflows/recovery.md +++ b/docs/versioned_docs/version-2.11/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.11/workflows/sbom.md b/docs/versioned_docs/version-2.11/workflows/sbom.md index 9ef6eb65c..6c1702dee 100644 --- a/docs/versioned_docs/version-2.11/workflows/sbom.md +++ b/docs/versioned_docs/version-2.11/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -11,13 +11,15 @@ SBOMs for Constellation are generated using [Syft](https://github.com/anchore/sy :::note The public key for Edgeless Systems' long-term code-signing key is: + ``` -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at https://edgeless.systems/es.pub and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -38,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.11/workflows/scale.md b/docs/versioned_docs/version-2.11/workflows/scale.md index 06898ad0c..63b727c7d 100644 --- a/docs/versioned_docs/version-2.11/workflows/scale.md +++ b/docs/versioned_docs/version-2.11/workflows/scale.md @@ -51,30 +51,30 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the worker ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + ## Control-plane node scaling @@ -82,30 +82,30 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.11/workflows/storage.md b/docs/versioned_docs/version-2.11/workflows/storage.md index 9e3d96346..06fbc4de6 100644 --- a/docs/versioned_docs/version-2.11/workflows/storage.md +++ b/docs/versioned_docs/version-2.11/workflows/storage.md @@ -21,30 +21,30 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. - - + + **Constellation CSI driver for AWS Elastic Block Store** Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -53,8 +53,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -82,8 +82,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -111,8 +111,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + AWS comes with two storage classes by default. @@ -140,8 +140,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/docs/versioned_docs/version-2.11/workflows/terminate.md b/docs/versioned_docs/version-2.11/workflows/terminate.md index 647eadb42..f33489ca5 100644 --- a/docs/versioned_docs/version-2.11/workflows/terminate.md +++ b/docs/versioned_docs/version-2.11/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.11/workflows/troubleshooting.md b/docs/versioned_docs/version-2.11/workflows/troubleshooting.md index a3e25a0fe..c40e6496e 100644 --- a/docs/versioned_docs/version-2.11/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.11/workflows/troubleshooting.md @@ -101,8 +101,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -112,8 +112,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -128,16 +128,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.11/workflows/trusted-launch.md b/docs/versioned_docs/version-2.11/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.11/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.11/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.11/workflows/verify-cli.md b/docs/versioned_docs/version-2.11/workflows/verify-cli.md index 1280c51b0..e33569d37 100644 --- a/docs/versioned_docs/version-2.11/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.11/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,11 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.12/architecture/attestation.md b/docs/versioned_docs/version-2.12/architecture/attestation.md index e37533995..f9c9ac38e 100644 --- a/docs/versioned_docs/version-2.12/architecture/attestation.md +++ b/docs/versioned_docs/version-2.12/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,16 +217,16 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,18 +248,18 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + There is no additional configuration available for GCP. - - + + There is no additional configuration available for AWS. - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.12/architecture/keys.md b/docs/versioned_docs/version-2.12/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.12/architecture/keys.md +++ b/docs/versioned_docs/version-2.12/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.12/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.12/getting-started/first-steps-local.md index de9c66e9b..a6e825906 100644 --- a/docs/versioned_docs/version-2.12/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.12/getting-started/first-steps-local.md @@ -45,8 +45,8 @@ sudo iptables -P FORWARD ACCEPT ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -74,8 +74,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -151,8 +151,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -205,8 +205,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -217,8 +217,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -246,8 +246,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.12/getting-started/first-steps.md b/docs/versioned_docs/version-2.12/getting-started/first-steps.md index 07b7f8410..9ebe21701 100644 --- a/docs/versioned_docs/version-2.12/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.12/getting-started/first-steps.md @@ -15,39 +15,39 @@ If you encounter any problem with the following steps, make sure to use the [lat 1. Create the [configuration file](../workflows/config.md) for your cloud provider. - + - + ```bash constellation config generate azure ``` - + - + ```bash constellation config generate gcp ``` - + - + ```bash constellation config generate aws ``` - + - + 2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config @@ -62,21 +62,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `westeurope` * `southeastasia` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --update-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config @@ -103,8 +103,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.12/getting-started/install.md b/docs/versioned_docs/version-2.12/getting-started/install.md index 03848d23b..2fabcf0b1 100644 --- a/docs/versioned_docs/version-2.12/getting-started/install.md +++ b/docs/versioned_docs/version-2.12/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,39 +105,42 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` -* `Microsoft.Network/natGateways/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` +- `Microsoft.Network/natGateways/*` The built-in `Contributor` role is a superset of these permissions. @@ -148,94 +148,94 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.firewalls.update` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroupManagers.update` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.update` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.firewalls.update` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroupManagers.update` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.update` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -281,13 +281,12 @@ The built-in `AdministratorAccess` policy is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). - The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -297,8 +296,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -314,8 +313,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -338,8 +337,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -355,10 +354,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.12/overview/clouds.md b/docs/versioned_docs/version-2.12/overview/clouds.md index 3ccbb0d6d..dfc3d5307 100644 --- a/docs/versioned_docs/version-2.12/overview/clouds.md +++ b/docs/versioned_docs/version-2.12/overview/clouds.md @@ -31,11 +31,11 @@ This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. diff --git a/docs/versioned_docs/version-2.12/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.12/overview/confidential-kubernetes.md index ca20df4de..bff8c3322 100644 --- a/docs/versioned_docs/version-2.12/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.12/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.12/overview/performance/compute.md b/docs/versioned_docs/version-2.12/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.12/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.12/overview/performance/io.md b/docs/versioned_docs/version-2.12/overview/performance/io.md index dc7cf3d8b..3ae796f8a 100644 --- a/docs/versioned_docs/version-2.12/overview/performance/io.md +++ b/docs/versioned_docs/version-2.12/overview/performance/io.md @@ -58,7 +58,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. diff --git a/docs/versioned_docs/version-2.12/overview/performance/performance.md b/docs/versioned_docs/version-2.12/overview/performance/performance.md index 7f22a693e..59bf86602 100644 --- a/docs/versioned_docs/version-2.12/overview/performance/performance.md +++ b/docs/versioned_docs/version-2.12/overview/performance/performance.md @@ -1,18 +1,10 @@ # Performance analysis of Constellation -This section provides a comprehensive examination of the performance characteristics of Constellation, encompassing various aspects, including runtime encryption, I/O benchmarks, and real-world applications. +This section provides a comprehensive examination of the performance characteristics of Constellation. -## Impact of runtime encryption on performance +## Runtime encryption -All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. - -### AMD and Azure benchmarking - -AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. - -### AMD and Google benchmarking - -Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. ## I/O performance benchmarks diff --git a/docs/versioned_docs/version-2.12/overview/product.md b/docs/versioned_docs/version-2.12/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.12/overview/product.md +++ b/docs/versioned_docs/version-2.12/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.12/workflows/config.md b/docs/versioned_docs/version-2.12/workflows/config.md index 95f95aeec..edc3c9091 100644 --- a/docs/versioned_docs/version-2.12/workflows/config.md +++ b/docs/versioned_docs/version-2.12/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,49 +14,49 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. @@ -75,8 +75,8 @@ AWS is currently investigating the issue. SNP-based attestation will be enabled as soon as a fix is verified. ::: - - + + Fill the desired VM type into the **instanceType** fields in the `constellation-conf.yml` file. @@ -86,7 +86,6 @@ By default, Constellation creates the node groups `control_plane_default` and `w If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. Each node group can be scaled individually. - Consider the following example for AWS: ```yaml @@ -120,9 +119,9 @@ You can use the field `zone` to specify what availability zone nodes of the grou On Azure, this field is empty by default and nodes are automatically spread across availability zones. Consult the documentation of your cloud provider for more information: -- [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) -- [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) -- [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) ## Choosing a Kubernetes version @@ -134,8 +133,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -155,23 +154,23 @@ Note that CVMs are currently only supported in a few regions, check [Azure's pro Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -195,16 +194,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -233,19 +232,19 @@ The following describes the configuration fields and how you obtain the required The user-assigned identity is used by instances of the cluster to access other cloud resources. For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -259,9 +258,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -292,9 +291,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.12/workflows/create.md b/docs/versioned_docs/version-2.12/workflows/create.md index ab7bf80ea..eccb2699a 100644 --- a/docs/versioned_docs/version-2.12/workflows/create.md +++ b/docs/versioned_docs/version-2.12/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + ```bash constellation create @@ -35,8 +35,8 @@ constellation create *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -77,8 +77,8 @@ yq eval '.infrastructure.initSecret ="$CONSTELL_INIT_SECRET"' --inplace constell yq eval '.infrastructure.clusterEndpoint ="$CONSTELL_IP"' --inplace constellation-state.yaml ``` - - + + ## The *init* step @@ -96,6 +96,6 @@ export KUBECONFIG="$PWD/constellation-admin.conf" 🏁 That's it. You've successfully created a Constellation cluster. - ### Troubleshooting + In case `init` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.12/workflows/recovery.md b/docs/versioned_docs/version-2.12/workflows/recovery.md index 955981749..f2d5f22c1 100644 --- a/docs/versioned_docs/version-2.12/workflows/recovery.md +++ b/docs/versioned_docs/version-2.12/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.12/workflows/sbom.md b/docs/versioned_docs/version-2.12/workflows/sbom.md index 9ef6eb65c..6c1702dee 100644 --- a/docs/versioned_docs/version-2.12/workflows/sbom.md +++ b/docs/versioned_docs/version-2.12/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -11,13 +11,15 @@ SBOMs for Constellation are generated using [Syft](https://github.com/anchore/sy :::note The public key for Edgeless Systems' long-term code-signing key is: + ``` -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at https://edgeless.systems/es.pub and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -38,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.12/workflows/scale.md b/docs/versioned_docs/version-2.12/workflows/scale.md index 06898ad0c..63b727c7d 100644 --- a/docs/versioned_docs/version-2.12/workflows/scale.md +++ b/docs/versioned_docs/version-2.12/workflows/scale.md @@ -51,30 +51,30 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the worker ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + ## Control-plane node scaling @@ -82,30 +82,30 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.12/workflows/storage.md b/docs/versioned_docs/version-2.12/workflows/storage.md index 9e3d96346..06fbc4de6 100644 --- a/docs/versioned_docs/version-2.12/workflows/storage.md +++ b/docs/versioned_docs/version-2.12/workflows/storage.md @@ -21,30 +21,30 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. - - + + **Constellation CSI driver for AWS Elastic Block Store** Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -53,8 +53,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -82,8 +82,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -111,8 +111,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + AWS comes with two storage classes by default. @@ -140,8 +140,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/docs/versioned_docs/version-2.12/workflows/terminate.md b/docs/versioned_docs/version-2.12/workflows/terminate.md index 14a130d55..af7dbc1db 100644 --- a/docs/versioned_docs/version-2.12/workflows/terminate.md +++ b/docs/versioned_docs/version-2.12/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-state.yaml constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.12/workflows/troubleshooting.md b/docs/versioned_docs/version-2.12/workflows/troubleshooting.md index a3e25a0fe..c40e6496e 100644 --- a/docs/versioned_docs/version-2.12/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.12/workflows/troubleshooting.md @@ -101,8 +101,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -112,8 +112,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -128,16 +128,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.12/workflows/trusted-launch.md b/docs/versioned_docs/version-2.12/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.12/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.12/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.12/workflows/verify-cli.md b/docs/versioned_docs/version-2.12/workflows/verify-cli.md index 1280c51b0..e33569d37 100644 --- a/docs/versioned_docs/version-2.12/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.12/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,11 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.13/architecture/attestation.md b/docs/versioned_docs/version-2.13/architecture/attestation.md index 576bc8865..8408cc5f0 100644 --- a/docs/versioned_docs/version-2.13/architecture/attestation.md +++ b/docs/versioned_docs/version-2.13/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,16 +217,16 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,18 +248,18 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + There is no additional configuration available for GCP. - - + + There is no additional configuration available for AWS. - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.13/architecture/keys.md b/docs/versioned_docs/version-2.13/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.13/architecture/keys.md +++ b/docs/versioned_docs/version-2.13/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.13/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.13/getting-started/first-steps-local.md index 90a7317af..890a12654 100644 --- a/docs/versioned_docs/version-2.13/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.13/getting-started/first-steps-local.md @@ -45,8 +45,8 @@ sudo iptables -P FORWARD ACCEPT ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -74,8 +74,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -152,8 +152,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -206,8 +206,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -218,8 +218,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -247,8 +247,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.13/getting-started/first-steps.md b/docs/versioned_docs/version-2.13/getting-started/first-steps.md index 040be5478..3ec110064 100644 --- a/docs/versioned_docs/version-2.13/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.13/getting-started/first-steps.md @@ -15,39 +15,39 @@ If you encounter any problem with the following steps, make sure to use the [lat 1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. - + - + ```bash constellation config generate azure ``` - + - + ```bash constellation config generate gcp ``` - + - + ```bash constellation config generate aws ``` - + - + 2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config @@ -62,21 +62,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `westeurope` * `southeastasia` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --update-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config @@ -103,8 +103,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.13/getting-started/install.md b/docs/versioned_docs/version-2.13/getting-started/install.md index 03848d23b..2fabcf0b1 100644 --- a/docs/versioned_docs/version-2.13/getting-started/install.md +++ b/docs/versioned_docs/version-2.13/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,39 +105,42 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` -* `Microsoft.Network/natGateways/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` +- `Microsoft.Network/natGateways/*` The built-in `Contributor` role is a superset of these permissions. @@ -148,94 +148,94 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.firewalls.update` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroupManagers.update` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.update` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.firewalls.update` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroupManagers.update` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.update` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -281,13 +281,12 @@ The built-in `AdministratorAccess` policy is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). - The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -297,8 +296,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -314,8 +313,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -338,8 +337,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -355,10 +354,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.13/overview/clouds.md b/docs/versioned_docs/version-2.13/overview/clouds.md index 3ccbb0d6d..dfc3d5307 100644 --- a/docs/versioned_docs/version-2.13/overview/clouds.md +++ b/docs/versioned_docs/version-2.13/overview/clouds.md @@ -31,11 +31,11 @@ This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. diff --git a/docs/versioned_docs/version-2.13/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.13/overview/confidential-kubernetes.md index ca20df4de..bff8c3322 100644 --- a/docs/versioned_docs/version-2.13/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.13/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.13/overview/performance/compute.md b/docs/versioned_docs/version-2.13/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.13/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.13/overview/performance/io.md b/docs/versioned_docs/version-2.13/overview/performance/io.md index dc7cf3d8b..3ae796f8a 100644 --- a/docs/versioned_docs/version-2.13/overview/performance/io.md +++ b/docs/versioned_docs/version-2.13/overview/performance/io.md @@ -58,7 +58,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. diff --git a/docs/versioned_docs/version-2.13/overview/performance/performance.md b/docs/versioned_docs/version-2.13/overview/performance/performance.md index 7f22a693e..59bf86602 100644 --- a/docs/versioned_docs/version-2.13/overview/performance/performance.md +++ b/docs/versioned_docs/version-2.13/overview/performance/performance.md @@ -1,18 +1,10 @@ # Performance analysis of Constellation -This section provides a comprehensive examination of the performance characteristics of Constellation, encompassing various aspects, including runtime encryption, I/O benchmarks, and real-world applications. +This section provides a comprehensive examination of the performance characteristics of Constellation. -## Impact of runtime encryption on performance +## Runtime encryption -All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. - -### AMD and Azure benchmarking - -AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. - -### AMD and Google benchmarking - -Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. ## I/O performance benchmarks diff --git a/docs/versioned_docs/version-2.13/overview/product.md b/docs/versioned_docs/version-2.13/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.13/overview/product.md +++ b/docs/versioned_docs/version-2.13/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.13/workflows/config.md b/docs/versioned_docs/version-2.13/workflows/config.md index 95f95aeec..edc3c9091 100644 --- a/docs/versioned_docs/version-2.13/workflows/config.md +++ b/docs/versioned_docs/version-2.13/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,49 +14,49 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. @@ -75,8 +75,8 @@ AWS is currently investigating the issue. SNP-based attestation will be enabled as soon as a fix is verified. ::: - - + + Fill the desired VM type into the **instanceType** fields in the `constellation-conf.yml` file. @@ -86,7 +86,6 @@ By default, Constellation creates the node groups `control_plane_default` and `w If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. Each node group can be scaled individually. - Consider the following example for AWS: ```yaml @@ -120,9 +119,9 @@ You can use the field `zone` to specify what availability zone nodes of the grou On Azure, this field is empty by default and nodes are automatically spread across availability zones. Consult the documentation of your cloud provider for more information: -- [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) -- [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) -- [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) ## Choosing a Kubernetes version @@ -134,8 +133,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -155,23 +154,23 @@ Note that CVMs are currently only supported in a few regions, check [Azure's pro Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -195,16 +194,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -233,19 +232,19 @@ The following describes the configuration fields and how you obtain the required The user-assigned identity is used by instances of the cluster to access other cloud resources. For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -259,9 +258,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -292,9 +291,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.13/workflows/create.md b/docs/versioned_docs/version-2.13/workflows/create.md index 605413cac..f347e9f27 100644 --- a/docs/versioned_docs/version-2.13/workflows/create.md +++ b/docs/versioned_docs/version-2.13/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + ```bash constellation create @@ -35,8 +35,8 @@ constellation create *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. This provides flexibility in DevOps and can meet potential regulatory requirements. @@ -55,7 +55,7 @@ management tooling of your choice. You need to keep the essential functionality :::info - On Azure, if the enforcement policy is set to `MAAFallback` in `constellation-config.yaml`, a manual update to the MAA provider's policy is necessary. + On Azure, a manual update to the MAA provider's policy is necessary. You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestationURL | jq -r)`, when using the minimal Terraform configuration). ```bash @@ -72,8 +72,8 @@ Fill these outputs into the corresponding fields of the `Infrastructure` block i Continue with [initializing your cluster](#the-apply-step). - - + + ## The *apply* step diff --git a/docs/versioned_docs/version-2.13/workflows/recovery.md b/docs/versioned_docs/version-2.13/workflows/recovery.md index 955981749..f2d5f22c1 100644 --- a/docs/versioned_docs/version-2.13/workflows/recovery.md +++ b/docs/versioned_docs/version-2.13/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.13/workflows/sbom.md b/docs/versioned_docs/version-2.13/workflows/sbom.md index 9ef6eb65c..6c1702dee 100644 --- a/docs/versioned_docs/version-2.13/workflows/sbom.md +++ b/docs/versioned_docs/version-2.13/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -11,13 +11,15 @@ SBOMs for Constellation are generated using [Syft](https://github.com/anchore/sy :::note The public key for Edgeless Systems' long-term code-signing key is: + ``` -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at https://edgeless.systems/es.pub and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -38,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.13/workflows/scale.md b/docs/versioned_docs/version-2.13/workflows/scale.md index 06898ad0c..63b727c7d 100644 --- a/docs/versioned_docs/version-2.13/workflows/scale.md +++ b/docs/versioned_docs/version-2.13/workflows/scale.md @@ -51,30 +51,30 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the worker ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + ## Control-plane node scaling @@ -82,30 +82,30 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.13/workflows/storage.md b/docs/versioned_docs/version-2.13/workflows/storage.md index 9e3d96346..06fbc4de6 100644 --- a/docs/versioned_docs/version-2.13/workflows/storage.md +++ b/docs/versioned_docs/version-2.13/workflows/storage.md @@ -21,30 +21,30 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. - - + + **Constellation CSI driver for AWS Elastic Block Store** Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -53,8 +53,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -82,8 +82,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -111,8 +111,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + AWS comes with two storage classes by default. @@ -140,8 +140,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/docs/versioned_docs/version-2.13/workflows/terminate.md b/docs/versioned_docs/version-2.13/workflows/terminate.md index 062214c1c..e9599cb2b 100644 --- a/docs/versioned_docs/version-2.13/workflows/terminate.md +++ b/docs/versioned_docs/version-2.13/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-state.yaml constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.13/workflows/terraform-module.md b/docs/versioned_docs/version-2.13/workflows/terraform-module.md index e0534b9f5..e38a4cc2c 100644 --- a/docs/versioned_docs/version-2.13/workflows/terraform-module.md +++ b/docs/versioned_docs/version-2.13/workflows/terraform-module.md @@ -1,12 +1,15 @@ # Use the Terraform module + You can manage a Constellation cluster through Terraform. The module package is available as part of the [GitHub release](https://github.com/edgelesssys/constellation/releases/). It consists of a convenience module for each cloud service provider (`{csp}-constellation`) that combines the IAM (`infrastructure/{csp}/iam`), infrastructure (`infrastructure/{csp}`), and constellation (`constellation-cluster`) modules. ## Prerequisites + - a Linux / Mac operating system - a Terraform installation of version `v1.4.4` or above ## Quick setup + The convenience module allows setting up a Constellation cluster with a single module. It's easiest to consume the module through a remote source, as shown below. This allows to upgrade the cluster to a newer Constellation version by simply updating the module source. @@ -18,6 +21,7 @@ The files are deleted on `terraform destroy`. ::: 1. Create a directory (workspace) for your Constellation cluster. + ```bash mkdir constellation-workspace cd constellation-workspace @@ -25,9 +29,9 @@ The files are deleted on `terraform destroy`. 1. Create a `main.tf` file to call the CSP specific Constellation module. - + - + ``` module "azure-constellation" { @@ -55,9 +59,9 @@ The files are deleted on `terraform destroy`. } ``` - + - + ``` module "aws-constellation" { @@ -86,9 +90,9 @@ The files are deleted on `terraform destroy`. } ``` - + - + ``` module "gcp-constellation" { @@ -96,11 +100,11 @@ The files are deleted on `terraform destroy`. name = "constell" project = "constell-proj" // replace with your project id service_account_id = "constid" - zone = "europe-west2-a" + zone = "europe-west3-a" node_groups = { control_plane_default = { role = "control-plane" - zone = "europe-west2-a" + zone = "europe-west3-a" instance_type = "n2d-standard-4" disk_size = 30 disk_type = "pd-ssd" @@ -108,7 +112,7 @@ The files are deleted on `terraform destroy`. }, worker_default = { role = "worker" - zone = "europe-west2-a" + zone = "europe-west3-a" instance_type = "n2d-standard-4" disk_size = 30 disk_type = "pd-ssd" @@ -118,25 +122,29 @@ The files are deleted on `terraform destroy`. } ``` - - + + 3. Initialize and apply the module. + ```bash terraform init terraform apply ``` ## Custom setup + If you need to separate IAM and cluster management or need custom infrastructure, you can also call the submodules individually. Look at the respective convenience module (`{csp}-constellation`) for how you can structure the module calls. The submodules are: + - `constellation-cluster`: manages the Constellation cluster - `fetch-image`: translates the Constellation image version to the image ID of the cloud service provider - `infrastructure/{csp}`: contains the cluster infrastructure resources - `infrastructure/iam/{csp}`: contains the IAM resources used within the cluster ## Cluster upgrades + :::tip For general information on cluster upgrades, see [Upgrade your cluster](./upgrade.md). ::: @@ -145,6 +153,7 @@ Using a [remote address as module source](https://developer.hashicorp.com/terraf 1. Update the `` variable inside the `source` field of the module. 2. Upgrade the Terraform module and provider dependencies and apply the Constellation upgrade. + ```bash terraform init -upgrade terraform apply diff --git a/docs/versioned_docs/version-2.13/workflows/troubleshooting.md b/docs/versioned_docs/version-2.13/workflows/troubleshooting.md index 05f515ed7..e8220c9a2 100644 --- a/docs/versioned_docs/version-2.13/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.13/workflows/troubleshooting.md @@ -101,8 +101,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -112,8 +112,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -128,16 +128,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.13/workflows/trusted-launch.md b/docs/versioned_docs/version-2.13/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.13/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.13/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.13/workflows/verify-cli.md b/docs/versioned_docs/version-2.13/workflows/verify-cli.md index 1280c51b0..e33569d37 100644 --- a/docs/versioned_docs/version-2.13/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.13/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,11 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.14/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.14/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.14/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.14/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.14/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.14/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.14/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.14/_media/concept-constellation.svg b/docs/versioned_docs/version-2.14/_media/concept-constellation.svg new file mode 100644 index 000000000..caa7f847d --- /dev/null +++ b/docs/versioned_docs/version-2.14/_media/concept-constellation.svg @@ -0,0 +1,657 @@ + + diff --git a/docs/versioned_docs/version-2.14/_media/concept-managed.svg b/docs/versioned_docs/version-2.14/_media/concept-managed.svg new file mode 100644 index 000000000..718412aad --- /dev/null +++ b/docs/versioned_docs/version-2.14/_media/concept-managed.svg @@ -0,0 +1,847 @@ + + diff --git a/docs/versioned_docs/version-2.14/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.14/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.14/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.14/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.14/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.14/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.14/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/example-online-boutique.jpg differ diff --git a/docs/docs/_media/product-overview-dark.png b/docs/versioned_docs/version-2.14/_media/product-overview-dark.png similarity index 100% rename from docs/docs/_media/product-overview-dark.png rename to docs/versioned_docs/version-2.14/_media/product-overview-dark.png diff --git a/docs/docs/_media/product-overview.png b/docs/versioned_docs/version-2.14/_media/product-overview.png similarity index 100% rename from docs/docs/_media/product-overview.png rename to docs/versioned_docs/version-2.14/_media/product-overview.png diff --git a/docs/versioned_docs/version-2.14/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.14/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.14/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.14/_media/tcb.svg b/docs/versioned_docs/version-2.14/_media/tcb.svg new file mode 100644 index 000000000..f692ffd0e --- /dev/null +++ b/docs/versioned_docs/version-2.14/_media/tcb.svg @@ -0,0 +1,1287 @@ + + diff --git a/docs/versioned_docs/version-2.14/architecture/attestation.md b/docs/versioned_docs/version-2.14/architecture/attestation.md new file mode 100644 index 000000000..415b41f47 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/attestation.md @@ -0,0 +1,333 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, it's usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, it uses measured boot for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection, guaranteeing forward integrity. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the measured boot's PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +There is no additional configuration available for GCP. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM and it's vTPM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Chain of trust + +So far, this page described how an entire Constellation cluster can be verified using hardware attestation capabilities and runtime measurements. +The last missing link is how the ground truth in the form of runtime measurements can be securely distributed to the verifying party. + +The build process of Constellation images also creates the ground truth runtime measurements. +With every release, Edgeless Systems publishes signed runtime measurements. + +The CLI executable is also signed by Edgeless Systems. +You can [verify its signature](../workflows/verify-cli.md). + +The CLI contains the public key required to verify signed runtime measurements from Edgeless Systems. +When a cluster is [created](../workflows/create.md) or [upgraded](../workflows/upgrade.md), the CLI automatically verifies the measurements for the selected image. + +Thus, there's a chain of trust based on cryptographic signatures, which goes from CLI to runtime measurements to images. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[Edgeless]-- "signs (cosign)" -->B[CLI] + C[User]-- "verifies (cosign)" -->B[CLI] + B[CLI]-- "contains" -->D["Public Key"] + A[Edgeless]-- "signs" -->E["Runtime measurements"] + D["Public key"]-- "verifies" -->E["Runtime measurements"] + E["Runtime measurements"]-- "verify" -->F["Constellation cluster"] +``` + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.14/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.14/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.14/architecture/images.md b/docs/versioned_docs/version-2.14/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.14/architecture/keys.md b/docs/versioned_docs/version-2.14/architecture/keys.md new file mode 100644 index 000000000..553d9d4e2 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/keys.md @@ -0,0 +1,131 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). +Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.14/architecture/microservices.md b/docs/versioned_docs/version-2.14/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.14/architecture/networking.md b/docs/versioned_docs/version-2.14/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.14/architecture/observability.md b/docs/versioned_docs/version-2.14/architecture/observability.md new file mode 100644 index 000000000..39018e6b1 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/observability.md @@ -0,0 +1,78 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Constellation uses cloud logging for events occurring during the early stages of a node's boot process. +These logs include [Bootstrapper](./microservices.md#bootstrapper) events and [state disk UUIDs](../architecture/images.md#state-disk). +You can access the cloud logging [directly via the cloud provider endpoints](../workflows/troubleshooting.md#cloud-logging). + +More detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.14/architecture/orchestration.md b/docs/versioned_docs/version-2.14/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.14/architecture/overview.md b/docs/versioned_docs/version-2.14/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.14/architecture/versions.md b/docs/versioned_docs/version-2.14/architecture/versions.md new file mode 100644 index 000000000..6ee17873c --- /dev/null +++ b/docs/versioned_docs/version-2.14/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.26.11 +* v1.27.8 +* v1.28.4 diff --git a/docs/versioned_docs/version-2.14/getting-started/examples.md b/docs/versioned_docs/version-2.14/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.14/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.14/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.14/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.14/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.14/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.14/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.14/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.14/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.14/getting-started/first-steps.md b/docs/versioned_docs/version-2.14/getting-started/first-steps.md new file mode 100644 index 000000000..e4e6c8e48 --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/first-steps.md @@ -0,0 +1,215 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. + + + + + + ```bash + constellation config generate azure + ``` + + + + + + ```bash + constellation config generate gcp + ``` + + + + + + ```bash + constellation config generate aws + ``` + + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + + ```bash + constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that CVMs are currently only supported in a few regions, check [Azure's products available by region](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). These are: + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.14/getting-started/install.md b/docs/versioned_docs/version-2.14/getting-started/install.md new file mode 100644 index 000000000..08e2315ef --- /dev/null +++ b/docs/versioned_docs/version-2.14/getting-started/install.md @@ -0,0 +1,367 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux or macOS +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.14/intro.md b/docs/versioned_docs/version-2.14/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.14/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.14/overview/clouds.md b/docs/versioned_docs/version-2.14/overview/clouds.md new file mode 100644 index 000000000..dfc3d5307 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/clouds.md @@ -0,0 +1,58 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or, in the future, Intel TDX (available in Xeon CPUs from the Sapphire Rapids generation onward) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures as of June 2023. + +| **Feature** | **Azure** | **GCP** | **AWS** | **OpenStack (Yoga)** | +|-----------------------------------|-----------|---------|---------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | Depends on kernel/HV | +| **4. Reviewable firmware** | No* | No | Yes | Depends on kernel/HV | +| **5. Confidential measured boot** | Yes | No | No | Depends on kernel/HV | + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to remote-attestation statements. +The CVM firmware running in VM Privilege Level (VMPL) 0 provides a vTPM (5), but it's closed source (4). +This firmware is signed by Azure. +The signature is reflected in the remote-attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. +CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. +However, regarding (4), the CVMs still include closed-source firmware. + +Intel and Google have [collaborated](https://cloud.google.com/blog/products/identity-security/rsa-google-intel-confidential-computing-more-secure) to enhance the security of TDX, and have recently [revealed](https://venturebeat.com/security/intel-launches-confidential-computing-solution-for-virtual-machines/) their plans to make TDX compatible with Google Cloud. + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to remote-attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.14/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.14/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.14/overview/license.md b/docs/versioned_docs/version-2.14/overview/license.md new file mode 100644 index 000000000..9a2ddc8f6 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## Azure Marketplace + +Constellation is available through the Azure Marketplace. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your Azure account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.14/overview/performance/application.md b/docs/versioned_docs/version-2.14/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.14/overview/performance/compute.md b/docs/versioned_docs/version-2.14/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.14/overview/performance/io.md b/docs/versioned_docs/version-2.14/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.14/overview/performance/performance.md b/docs/versioned_docs/version-2.14/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.14/overview/product.md b/docs/versioned_docs/version-2.14/overview/product.md new file mode 100644 index 000000000..02e12e2f3 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.14/overview/security-benefits.md b/docs/versioned_docs/version-2.14/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.14/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.14/reference/cli.md b/docs/versioned_docs/version-2.14/reference/cli.md new file mode 100644 index 000000000..6a3e7c429 --- /dev/null +++ b/docs/versioned_docs/version-2.14/reference/cli.md @@ -0,0 +1,842 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-trustedlaunch|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.27") +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.14/reference/migration.md b/docs/versioned_docs/version-2.14/reference/migration.md new file mode 100644 index 000000000..36680eef6 --- /dev/null +++ b/docs/versioned_docs/version-2.14/reference/migration.md @@ -0,0 +1,85 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrating from Azure's service principal authentication to managed identity authentication + +- The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +- To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +- Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +- To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + + +## Migrating from CLI versions before 2.10 + +- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +- The global `nodeGroups` field was added. +- The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +- The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +- The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +- The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +- The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +- The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +- The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +- The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | +
+- The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + - To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + - To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.14/reference/slsa.md b/docs/versioned_docs/version-2.14/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.14/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.14/reference/terraform.md b/docs/versioned_docs/version-2.14/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.14/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.14/workflows/cert-manager.md b/docs/versioned_docs/version-2.14/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.14/workflows/config.md b/docs/versioned_docs/version-2.14/workflows/config.md new file mode 100644 index 000000000..5e938c29c --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/config.md @@ -0,0 +1,306 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate aws +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the **instanceType** fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +Note that CVMs are currently only supported in a few regions, check [Azure's products available by region](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). These are: + +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. CVMs are currently only supported in a few regions, check [Azure's products available by region](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). These are: + + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.14/workflows/create.md b/docs/versioned_docs/version-2.14/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.14/workflows/lb.md b/docs/versioned_docs/version-2.14/workflows/lb.md new file mode 100644 index 000000000..11e403237 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/lb.md @@ -0,0 +1,15 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancing Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: diff --git a/docs/versioned_docs/version-2.14/workflows/recovery.md b/docs/versioned_docs/version-2.14/workflows/recovery.md new file mode 100644 index 000000000..f2d5f22c1 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/recovery.md @@ -0,0 +1,148 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover --master-secret constellation-mastersecret.json +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.14/workflows/s3proxy.md b/docs/versioned_docs/version-2.14/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.14/workflows/sbom.md b/docs/versioned_docs/version-2.14/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.14/workflows/scale.md b/docs/versioned_docs/version-2.14/workflows/scale.md new file mode 100644 index 000000000..63b727c7d --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/scale.md @@ -0,0 +1,111 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.14/workflows/storage.md b/docs/versioned_docs/version-2.14/workflows/storage.md new file mode 100644 index 000000000..06fbc4de6 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/storage.md @@ -0,0 +1,245 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, and GCP. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, and GCE PD, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.14/workflows/terraform-provider.md b/docs/versioned_docs/version-2.14/workflows/terraform-provider.md new file mode 100644 index 000000000..e831ccc9e --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/terraform-provider.md @@ -0,0 +1,118 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + When creating a cluster on Azure, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you + can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.14/workflows/troubleshooting.md b/docs/versioned_docs/version-2.14/workflows/troubleshooting.md new file mode 100644 index 000000000..64d7d3355 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/troubleshooting.md @@ -0,0 +1,168 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Cloud logging + +To provide information during early stages of a node's boot process, Constellation logs messages to the log systems of the cloud providers. Since these offerings **aren't** confidential, only generic information without any sensitive values is stored. This provides administrators with a high-level understanding of the current state of a node. + +You can view this information in the following places: + + + + +1. In your Azure subscription find the Constellation resource group. +2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. +3. On the left-hand side go to `Logs`, which is located in the section `Monitoring`. + - Close the Queries page if it pops up. +5. In the query text field type in `traces`, and click `Run`. + +To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` + + + + +1. Select the project that hosts Constellation. +2. Go to the `Compute Engine` service. +3. On the right-hand side of a VM entry select `More Actions` (a stacked ellipsis) + - Select `View logs` + +To **find the disk UUIDs** use the following query: `resource.type="gce_instance" text_payload=~"Disk UUID:.*\n" logName=~".*/constellation-boot-log"` + +:::info + +Constellation uses the default bucket to store logs. Its [default retention period is 30 days](https://cloud.google.com/logging/quotas#logs_retention_periods). + +::: + + + + +1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) +2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) +3. Select the log group that matches the name of your cluster. +4. Select the log stream for control or worker type nodes. + + + + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.14/workflows/trusted-launch.md b/docs/versioned_docs/version-2.14/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.14/workflows/upgrade.md b/docs/versioned_docs/version-2.14/workflows/upgrade.md new file mode 100644 index 000000000..7348c0dbc --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting it's availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.14/workflows/verify-cli.md b/docs/versioned_docs/version-2.14/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.14/workflows/verify-cluster.md b/docs/versioned_docs/version-2.14/workflows/verify-cluster.md new file mode 100644 index 000000000..20d416790 --- /dev/null +++ b/docs/versioned_docs/version-2.14/workflows/verify-cluster.md @@ -0,0 +1,96 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.15/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.15/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.15/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.15/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.15/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.15/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.15/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.15/_media/concept-constellation.svg b/docs/versioned_docs/version-2.15/_media/concept-constellation.svg new file mode 100644 index 000000000..caa7f847d --- /dev/null +++ b/docs/versioned_docs/version-2.15/_media/concept-constellation.svg @@ -0,0 +1,657 @@ + + diff --git a/docs/versioned_docs/version-2.15/_media/concept-managed.svg b/docs/versioned_docs/version-2.15/_media/concept-managed.svg new file mode 100644 index 000000000..718412aad --- /dev/null +++ b/docs/versioned_docs/version-2.15/_media/concept-managed.svg @@ -0,0 +1,847 @@ + + diff --git a/docs/versioned_docs/version-2.15/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.15/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.15/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.15/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.15/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.15/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.15/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.15/_media/product-overview-dark.png b/docs/versioned_docs/version-2.15/_media/product-overview-dark.png new file mode 100644 index 000000000..4aab5f8bd Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/product-overview-dark.png differ diff --git a/docs/versioned_docs/version-2.15/_media/product-overview.png b/docs/versioned_docs/version-2.15/_media/product-overview.png new file mode 100644 index 000000000..c44e633cd Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/product-overview.png differ diff --git a/docs/versioned_docs/version-2.15/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.15/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.15/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.15/_media/tcb.svg b/docs/versioned_docs/version-2.15/_media/tcb.svg new file mode 100644 index 000000000..f692ffd0e --- /dev/null +++ b/docs/versioned_docs/version-2.15/_media/tcb.svg @@ -0,0 +1,1287 @@ + + diff --git a/docs/versioned_docs/version-2.15/architecture/attestation.md b/docs/versioned_docs/version-2.15/architecture/attestation.md new file mode 100644 index 000000000..286b2466d --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/attestation.md @@ -0,0 +1,333 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, it's usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, it uses measured boot for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection, guaranteeing forward integrity. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the measured boot's PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +There is no additional configuration available for GCP. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM and it's vTPM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Chain of trust + +So far, this page described how an entire Constellation cluster can be verified using hardware attestation capabilities and runtime measurements. +The last missing link is how the ground truth in the form of runtime measurements can be securely distributed to the verifying party. + +The build process of Constellation images also creates the ground truth runtime measurements. The builds of Constellation images are reproducible and the measurements of an image can be recalculated and verified by everyone. +With every release, Edgeless Systems publishes signed runtime measurements. + +The CLI executable is also signed by Edgeless Systems. +You can [verify its signature](../workflows/verify-cli.md). + +The CLI contains the public key required to verify signed runtime measurements from Edgeless Systems. +When a cluster is [created](../workflows/create.md) or [upgraded](../workflows/upgrade.md), the CLI automatically verifies the measurements for the selected image. + +Thus, there's a chain of trust based on cryptographic signatures, which goes from CLI to runtime measurements to images. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[Edgeless]-- "signs (cosign)" -->B[CLI] + C[User]-- "verifies (cosign)" -->B[CLI] + B[CLI]-- "contains" -->D["Public Key"] + A[Edgeless]-- "signs" -->E["Runtime measurements"] + D["Public key"]-- "verifies" -->E["Runtime measurements"] + E["Runtime measurements"]-- "verify" -->F["Constellation cluster"] +``` + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.15/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.15/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.15/architecture/images.md b/docs/versioned_docs/version-2.15/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.15/architecture/keys.md b/docs/versioned_docs/version-2.15/architecture/keys.md new file mode 100644 index 000000000..553d9d4e2 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/keys.md @@ -0,0 +1,131 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). +Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.15/architecture/microservices.md b/docs/versioned_docs/version-2.15/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.15/architecture/networking.md b/docs/versioned_docs/version-2.15/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.15/architecture/observability.md b/docs/versioned_docs/version-2.15/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.15/architecture/orchestration.md b/docs/versioned_docs/version-2.15/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.15/architecture/overview.md b/docs/versioned_docs/version-2.15/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.15/architecture/versions.md b/docs/versioned_docs/version-2.15/architecture/versions.md new file mode 100644 index 000000000..fbdda5a57 --- /dev/null +++ b/docs/versioned_docs/version-2.15/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.27.9 +* v1.28.5 +* v1.29.0 diff --git a/docs/versioned_docs/version-2.15/getting-started/examples.md b/docs/versioned_docs/version-2.15/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.15/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.15/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.15/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.15/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.15/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.15/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.15/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.15/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.15/getting-started/first-steps.md b/docs/versioned_docs/version-2.15/getting-started/first-steps.md new file mode 100644 index 000000000..8618fb843 --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/first-steps.md @@ -0,0 +1,221 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. + + + + + + ```bash + constellation config generate azure + ``` + + + + + + ```bash + constellation config generate gcp + ``` + + + + + + ```bash + constellation config generate aws + ``` + + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + + ```bash + constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.15/getting-started/install.md b/docs/versioned_docs/version-2.15/getting-started/install.md new file mode 100644 index 000000000..08e2315ef --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/install.md @@ -0,0 +1,367 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux or macOS +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.15/getting-started/marketplaces.md b/docs/versioned_docs/version-2.15/getting-started/marketplaces.md new file mode 100644 index 000000000..62fcff5b6 --- /dev/null +++ b/docs/versioned_docs/version-2.15/getting-started/marketplaces.md @@ -0,0 +1,39 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of Azure and GCP. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + +## Azure + +On Azure, Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. + +## GCP + +On GCP, to use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.15/intro.md b/docs/versioned_docs/version-2.15/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.15/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.15/overview/clouds.md b/docs/versioned_docs/version-2.15/overview/clouds.md new file mode 100644 index 000000000..934f6c710 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/clouds.md @@ -0,0 +1,63 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures as of June 2023. + +| **Feature** | **Azure** | **GCP** | **AWS** | **OpenStack (Yoga)** | +|-----------------------------------|-----------|---------|---------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | Depends on kernel/HV | +| **4. Reviewable firmware** | No* | No | Yes | Depends on kernel/HV | +| **5. Confidential measured boot** | Yes | No | No | Depends on kernel/HV | + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to remote-attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the remote-attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. +CVMs with [SEV-SNP enabled are in public preview](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev-snp). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +In the past, Intel and Google have [collaborated](https://cloud.google.com/blog/products/identity-security/rsa-google-intel-confidential-computing-more-secure) to enhance the security of TDX. +Recently, Google has announced a [private preview for TDX](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense?hl=en). +With TDX on Google, Constellation has a similar TCB and attestation flow as with the current SEV-SNP offering. + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to remote-attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.15/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.15/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.15/overview/license.md b/docs/versioned_docs/version-2.15/overview/license.md new file mode 100644 index 000000000..c1ea47f17 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of Azure and GCP. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.15/overview/performance/application.md b/docs/versioned_docs/version-2.15/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.15/overview/performance/compute.md b/docs/versioned_docs/version-2.15/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.15/overview/performance/io.md b/docs/versioned_docs/version-2.15/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.15/overview/performance/performance.md b/docs/versioned_docs/version-2.15/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.15/overview/product.md b/docs/versioned_docs/version-2.15/overview/product.md new file mode 100644 index 000000000..02e12e2f3 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.15/overview/security-benefits.md b/docs/versioned_docs/version-2.15/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.15/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.15/reference/cli.md b/docs/versioned_docs/version-2.15/reference/cli.md new file mode 100644 index 000000000..52391f3d1 --- /dev/null +++ b/docs/versioned_docs/version-2.15/reference/cli.md @@ -0,0 +1,842 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.28") +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.15/reference/migration.md b/docs/versioned_docs/version-2.15/reference/migration.md new file mode 100644 index 000000000..36680eef6 --- /dev/null +++ b/docs/versioned_docs/version-2.15/reference/migration.md @@ -0,0 +1,85 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrating from Azure's service principal authentication to managed identity authentication + +- The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +- To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +- Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +- To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + + +## Migrating from CLI versions before 2.10 + +- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +- The global `nodeGroups` field was added. +- The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +- The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +- The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +- The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +- The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +- The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +- The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +- The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | +
+- The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + - To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + - To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.15/reference/slsa.md b/docs/versioned_docs/version-2.15/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.15/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.15/reference/terraform.md b/docs/versioned_docs/version-2.15/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.15/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.15/workflows/cert-manager.md b/docs/versioned_docs/version-2.15/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.15/workflows/config.md b/docs/versioned_docs/version-2.15/workflows/config.md new file mode 100644 index 000000000..be9cc6a98 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/config.md @@ -0,0 +1,318 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate aws +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.15/workflows/create.md b/docs/versioned_docs/version-2.15/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.15/workflows/lb.md b/docs/versioned_docs/version-2.15/workflows/lb.md new file mode 100644 index 000000000..11e403237 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/lb.md @@ -0,0 +1,15 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancing Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: diff --git a/docs/versioned_docs/version-2.15/workflows/recovery.md b/docs/versioned_docs/version-2.15/workflows/recovery.md new file mode 100644 index 000000000..592ab74af --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/recovery.md @@ -0,0 +1,148 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover --master-secret constellation-mastersecret.json +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.15/workflows/s3proxy.md b/docs/versioned_docs/version-2.15/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.15/workflows/sbom.md b/docs/versioned_docs/version-2.15/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.15/workflows/scale.md b/docs/versioned_docs/version-2.15/workflows/scale.md new file mode 100644 index 000000000..63b727c7d --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/scale.md @@ -0,0 +1,111 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.15/workflows/storage.md b/docs/versioned_docs/version-2.15/workflows/storage.md new file mode 100644 index 000000000..06fbc4de6 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/storage.md @@ -0,0 +1,245 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, and GCP. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, and GCE PD, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.15/workflows/terraform-provider.md b/docs/versioned_docs/version-2.15/workflows/terraform-provider.md new file mode 100644 index 000000000..e831ccc9e --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/terraform-provider.md @@ -0,0 +1,118 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + When creating a cluster on Azure, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you + can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.15/workflows/troubleshooting.md b/docs/versioned_docs/version-2.15/workflows/troubleshooting.md new file mode 100644 index 000000000..3c952dddc --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/troubleshooting.md @@ -0,0 +1,133 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.15/workflows/trusted-launch.md b/docs/versioned_docs/version-2.15/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.15/workflows/upgrade.md b/docs/versioned_docs/version-2.15/workflows/upgrade.md new file mode 100644 index 000000000..7348c0dbc --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting it's availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.15/workflows/verify-cli.md b/docs/versioned_docs/version-2.15/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.15/workflows/verify-cluster.md b/docs/versioned_docs/version-2.15/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.15/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.16/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.16/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.16/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.16/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.16/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.16/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.16/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.16/_media/concept-constellation.svg b/docs/versioned_docs/version-2.16/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.16/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.16/_media/concept-managed.svg b/docs/versioned_docs/version-2.16/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.16/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.16/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.16/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.16/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.16/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.16/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.16/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.16/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.16/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.16/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.16/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.16/_media/tcb.svg b/docs/versioned_docs/version-2.16/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.16/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.16/architecture/attestation.md b/docs/versioned_docs/version-2.16/architecture/attestation.md new file mode 100644 index 000000000..a07b35e5a --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/attestation.md @@ -0,0 +1,394 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for GCP. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.16/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.16/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.16/architecture/images.md b/docs/versioned_docs/version-2.16/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.16/architecture/keys.md b/docs/versioned_docs/version-2.16/architecture/keys.md new file mode 100644 index 000000000..553d9d4e2 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/keys.md @@ -0,0 +1,131 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). +Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.16/architecture/microservices.md b/docs/versioned_docs/version-2.16/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.16/architecture/networking.md b/docs/versioned_docs/version-2.16/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.16/architecture/observability.md b/docs/versioned_docs/version-2.16/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.16/architecture/orchestration.md b/docs/versioned_docs/version-2.16/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.16/architecture/overview.md b/docs/versioned_docs/version-2.16/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.16/architecture/versions.md b/docs/versioned_docs/version-2.16/architecture/versions.md new file mode 100644 index 000000000..fbdda5a57 --- /dev/null +++ b/docs/versioned_docs/version-2.16/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.27.9 +* v1.28.5 +* v1.29.0 diff --git a/docs/versioned_docs/version-2.16/getting-started/examples.md b/docs/versioned_docs/version-2.16/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.16/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.16/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.16/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.16/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.16/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.16/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.16/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.16/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.16/getting-started/first-steps.md b/docs/versioned_docs/version-2.16/getting-started/first-steps.md new file mode 100644 index 000000000..738868551 --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/first-steps.md @@ -0,0 +1,229 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.16/getting-started/install.md b/docs/versioned_docs/version-2.16/getting-started/install.md new file mode 100644 index 000000000..d52e43476 --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/install.md @@ -0,0 +1,429 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file under `~/.config/openstack/clouds.yaml` (`%AppData%\openstack\clouds.yaml` on Windows) with the credentials from the User Access Token + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_STACKIT_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.16/getting-started/marketplaces.md b/docs/versioned_docs/version-2.16/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.16/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.16/intro.md b/docs/versioned_docs/version-2.16/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.16/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.16/overview/clouds.md b/docs/versioned_docs/version-2.16/overview/clouds.md new file mode 100644 index 000000000..5f19d8877 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/clouds.md @@ -0,0 +1,67 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures as of June 2023. + +| **Feature** | **Azure** | **GCP** | **AWS** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|-----------|---------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | No* | No | Yes | No | Depends on kernel/HV | +| **5. Confidential measured boot** | Yes | No | No | No | Depends on kernel/HV | + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to remote-attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the remote-attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. +CVMs with [SEV-SNP enabled are in public preview](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev-snp). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +In the past, Intel and Google have [collaborated](https://cloud.google.com/blog/products/identity-security/rsa-google-intel-confidential-computing-more-secure) to enhance the security of TDX. +Recently, Google has announced a [private preview for TDX](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense?hl=en). +With TDX on Google, Constellation has a similar TCB and attestation flow as with the current SEV-SNP offering. + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to remote-attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.16/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.16/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.16/overview/license.md b/docs/versioned_docs/version-2.16/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.16/overview/performance/application.md b/docs/versioned_docs/version-2.16/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.16/overview/performance/compute.md b/docs/versioned_docs/version-2.16/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.16/overview/performance/io.md b/docs/versioned_docs/version-2.16/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.16/overview/performance/performance.md b/docs/versioned_docs/version-2.16/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.16/overview/product.md b/docs/versioned_docs/version-2.16/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.16/overview/security-benefits.md b/docs/versioned_docs/version-2.16/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.16/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.16/reference/cli.md b/docs/versioned_docs/version-2.16/reference/cli.md new file mode 100644 index 000000000..52391f3d1 --- /dev/null +++ b/docs/versioned_docs/version-2.16/reference/cli.md @@ -0,0 +1,842 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.28") +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.16/reference/migration.md b/docs/versioned_docs/version-2.16/reference/migration.md new file mode 100644 index 000000000..36680eef6 --- /dev/null +++ b/docs/versioned_docs/version-2.16/reference/migration.md @@ -0,0 +1,85 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrating from Azure's service principal authentication to managed identity authentication + +- The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +- To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +- Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +- To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + + +## Migrating from CLI versions before 2.10 + +- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +- The global `nodeGroups` field was added. +- The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +- The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +- The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +- The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +- The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +- The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +- The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +- The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | +
+- The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + - To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + - To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.16/reference/slsa.md b/docs/versioned_docs/version-2.16/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.16/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.16/reference/terraform.md b/docs/versioned_docs/version-2.16/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.16/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.16/workflows/cert-manager.md b/docs/versioned_docs/version-2.16/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.16/workflows/config.md b/docs/versioned_docs/version-2.16/workflows/config.md new file mode 100644 index 000000000..11cf31cbd --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.16/workflows/create.md b/docs/versioned_docs/version-2.16/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.16/workflows/lb.md b/docs/versioned_docs/version-2.16/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.16/workflows/recovery.md b/docs/versioned_docs/version-2.16/workflows/recovery.md new file mode 100644 index 000000000..50cd7ee72 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover --master-secret constellation-mastersecret.json +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.16/workflows/s3proxy.md b/docs/versioned_docs/version-2.16/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.16/workflows/sbom.md b/docs/versioned_docs/version-2.16/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.16/workflows/scale.md b/docs/versioned_docs/version-2.16/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.16/workflows/storage.md b/docs/versioned_docs/version-2.16/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.16/workflows/terraform-provider.md b/docs/versioned_docs/version-2.16/workflows/terraform-provider.md new file mode 100644 index 000000000..ed8f46eda --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/terraform-provider.md @@ -0,0 +1,129 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + When creating a cluster on Azure, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you + can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.16/workflows/troubleshooting.md b/docs/versioned_docs/version-2.16/workflows/troubleshooting.md new file mode 100644 index 000000000..195bce1cc --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/troubleshooting.md @@ -0,0 +1,151 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.16/workflows/trusted-launch.md b/docs/versioned_docs/version-2.16/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.16/workflows/upgrade.md b/docs/versioned_docs/version-2.16/workflows/upgrade.md new file mode 100644 index 000000000..7348c0dbc --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting it's availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.16/workflows/verify-cli.md b/docs/versioned_docs/version-2.16/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.16/workflows/verify-cluster.md b/docs/versioned_docs/version-2.16/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.16/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.17/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.17/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.17/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.17/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.17/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.17/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.17/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.17/_media/concept-constellation.svg b/docs/versioned_docs/version-2.17/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.17/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.17/_media/concept-managed.svg b/docs/versioned_docs/version-2.17/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.17/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.17/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.17/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.17/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.17/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.17/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.17/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.17/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.17/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.17/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.17/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.17/_media/tcb.svg b/docs/versioned_docs/version-2.17/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.17/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.17/architecture/attestation.md b/docs/versioned_docs/version-2.17/architecture/attestation.md new file mode 100644 index 000000000..a07b35e5a --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/attestation.md @@ -0,0 +1,394 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for GCP. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.17/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.17/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.17/architecture/images.md b/docs/versioned_docs/version-2.17/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.17/architecture/keys.md b/docs/versioned_docs/version-2.17/architecture/keys.md new file mode 100644 index 000000000..553d9d4e2 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/keys.md @@ -0,0 +1,131 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). +Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.17/architecture/microservices.md b/docs/versioned_docs/version-2.17/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.17/architecture/networking.md b/docs/versioned_docs/version-2.17/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.17/architecture/observability.md b/docs/versioned_docs/version-2.17/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.17/architecture/orchestration.md b/docs/versioned_docs/version-2.17/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.17/architecture/overview.md b/docs/versioned_docs/version-2.17/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.17/architecture/versions.md b/docs/versioned_docs/version-2.17/architecture/versions.md new file mode 100644 index 000000000..85d35e6a9 --- /dev/null +++ b/docs/versioned_docs/version-2.17/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.28.11 +* v1.29.6 +* v1.30.2 diff --git a/docs/versioned_docs/version-2.17/getting-started/examples.md b/docs/versioned_docs/version-2.17/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.17/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.17/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.17/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.17/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.17/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.17/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.17/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.17/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.17/getting-started/first-steps.md b/docs/versioned_docs/version-2.17/getting-started/first-steps.md new file mode 100644 index 000000000..c31263ed3 --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/first-steps.md @@ -0,0 +1,229 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.17/getting-started/install.md b/docs/versioned_docs/version-2.17/getting-started/install.md new file mode 100644 index 000000000..d52e43476 --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/install.md @@ -0,0 +1,429 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file under `~/.config/openstack/clouds.yaml` (`%AppData%\openstack\clouds.yaml` on Windows) with the credentials from the User Access Token + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_STACKIT_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.17/getting-started/marketplaces.md b/docs/versioned_docs/version-2.17/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.17/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.17/intro.md b/docs/versioned_docs/version-2.17/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.17/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.17/overview/clouds.md b/docs/versioned_docs/version-2.17/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.17/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.17/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.17/overview/license.md b/docs/versioned_docs/version-2.17/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.17/overview/performance/application.md b/docs/versioned_docs/version-2.17/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.17/overview/performance/compute.md b/docs/versioned_docs/version-2.17/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.17/overview/performance/io.md b/docs/versioned_docs/version-2.17/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.17/overview/performance/performance.md b/docs/versioned_docs/version-2.17/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.17/overview/product.md b/docs/versioned_docs/version-2.17/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.17/overview/security-benefits.md b/docs/versioned_docs/version-2.17/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.17/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.17/reference/cli.md b/docs/versioned_docs/version-2.17/reference/cli.md new file mode 100644 index 000000000..a728474e7 --- /dev/null +++ b/docs/versioned_docs/version-2.17/reference/cli.md @@ -0,0 +1,843 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-es|gcp-sev-snp|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.17/reference/migration.md b/docs/versioned_docs/version-2.17/reference/migration.md new file mode 100644 index 000000000..36680eef6 --- /dev/null +++ b/docs/versioned_docs/version-2.17/reference/migration.md @@ -0,0 +1,85 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrating from Azure's service principal authentication to managed identity authentication + +- The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +- To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +- Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +- To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + + +## Migrating from CLI versions before 2.10 + +- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +- The global `nodeGroups` field was added. +- The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +- The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +- The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +- The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +- The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +- The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +- The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +- The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | +
+- The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + - To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + - To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.17/reference/slsa.md b/docs/versioned_docs/version-2.17/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.17/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.17/reference/terraform.md b/docs/versioned_docs/version-2.17/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.17/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.17/workflows/cert-manager.md b/docs/versioned_docs/version-2.17/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.17/workflows/config.md b/docs/versioned_docs/version-2.17/workflows/config.md new file mode 100644 index 000000000..11cf31cbd --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.17/workflows/create.md b/docs/versioned_docs/version-2.17/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.17/workflows/lb.md b/docs/versioned_docs/version-2.17/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.17/workflows/recovery.md b/docs/versioned_docs/version-2.17/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.17/workflows/s3proxy.md b/docs/versioned_docs/version-2.17/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.17/workflows/sbom.md b/docs/versioned_docs/version-2.17/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.17/workflows/scale.md b/docs/versioned_docs/version-2.17/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.17/workflows/storage.md b/docs/versioned_docs/version-2.17/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.17/workflows/terraform-provider.md b/docs/versioned_docs/version-2.17/workflows/terraform-provider.md new file mode 100644 index 000000000..ed8f46eda --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/terraform-provider.md @@ -0,0 +1,129 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + When creating a cluster on Azure, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you + can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.17/workflows/troubleshooting.md b/docs/versioned_docs/version-2.17/workflows/troubleshooting.md new file mode 100644 index 000000000..195bce1cc --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/troubleshooting.md @@ -0,0 +1,151 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.17/workflows/trusted-launch.md b/docs/versioned_docs/version-2.17/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.17/workflows/upgrade.md b/docs/versioned_docs/version-2.17/workflows/upgrade.md new file mode 100644 index 000000000..7348c0dbc --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting it's availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.17/workflows/verify-cli.md b/docs/versioned_docs/version-2.17/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.17/workflows/verify-cluster.md b/docs/versioned_docs/version-2.17/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.17/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.18/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.18/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.18/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.18/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.18/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.18/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.18/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.18/_media/concept-constellation.svg b/docs/versioned_docs/version-2.18/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.18/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.18/_media/concept-managed.svg b/docs/versioned_docs/version-2.18/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.18/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.18/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.18/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.18/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.18/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.18/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.18/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.18/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.18/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.18/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.18/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.18/_media/tcb.svg b/docs/versioned_docs/version-2.18/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.18/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.18/architecture/attestation.md b/docs/versioned_docs/version-2.18/architecture/attestation.md new file mode 100644 index 000000000..9bd157460 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/attestation.md @@ -0,0 +1,409 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.18/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.18/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.18/architecture/images.md b/docs/versioned_docs/version-2.18/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.18/architecture/keys.md b/docs/versioned_docs/version-2.18/architecture/keys.md new file mode 100644 index 000000000..553d9d4e2 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/keys.md @@ -0,0 +1,131 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). +Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.18/architecture/microservices.md b/docs/versioned_docs/version-2.18/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.18/architecture/networking.md b/docs/versioned_docs/version-2.18/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.18/architecture/observability.md b/docs/versioned_docs/version-2.18/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.18/architecture/orchestration.md b/docs/versioned_docs/version-2.18/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.18/architecture/overview.md b/docs/versioned_docs/version-2.18/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.18/architecture/versions.md b/docs/versioned_docs/version-2.18/architecture/versions.md new file mode 100644 index 000000000..30d9d28e2 --- /dev/null +++ b/docs/versioned_docs/version-2.18/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.28.13 +* v1.29.8 +* v1.30.4 diff --git a/docs/versioned_docs/version-2.18/getting-started/examples.md b/docs/versioned_docs/version-2.18/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.18/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.18/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.18/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.18/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.18/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.18/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.18/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.18/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.18/getting-started/first-steps.md b/docs/versioned_docs/version-2.18/getting-started/first-steps.md new file mode 100644 index 000000000..18b723565 --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/first-steps.md @@ -0,0 +1,229 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.18/getting-started/install.md b/docs/versioned_docs/version-2.18/getting-started/install.md new file mode 100644 index 000000000..d52e43476 --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/install.md @@ -0,0 +1,429 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file under `~/.config/openstack/clouds.yaml` (`%AppData%\openstack\clouds.yaml` on Windows) with the credentials from the User Access Token + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_STACKIT_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.18/getting-started/marketplaces.md b/docs/versioned_docs/version-2.18/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.18/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.18/intro.md b/docs/versioned_docs/version-2.18/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.18/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.18/overview/clouds.md b/docs/versioned_docs/version-2.18/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.18/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.18/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.18/overview/license.md b/docs/versioned_docs/version-2.18/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.18/overview/performance/application.md b/docs/versioned_docs/version-2.18/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.18/overview/performance/compute.md b/docs/versioned_docs/version-2.18/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.18/overview/performance/io.md b/docs/versioned_docs/version-2.18/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.18/overview/performance/performance.md b/docs/versioned_docs/version-2.18/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.18/overview/product.md b/docs/versioned_docs/version-2.18/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.18/overview/security-benefits.md b/docs/versioned_docs/version-2.18/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.18/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.18/reference/cli.md b/docs/versioned_docs/version-2.18/reference/cli.md new file mode 100644 index 000000000..99acef520 --- /dev/null +++ b/docs/versioned_docs/version-2.18/reference/cli.md @@ -0,0 +1,844 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.18/reference/migration.md b/docs/versioned_docs/version-2.18/reference/migration.md new file mode 100644 index 000000000..36680eef6 --- /dev/null +++ b/docs/versioned_docs/version-2.18/reference/migration.md @@ -0,0 +1,85 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrating from Azure's service principal authentication to managed identity authentication + +- The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +- To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +- Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +- To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + + +## Migrating from CLI versions before 2.10 + +- AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +- The global `nodeGroups` field was added. +- The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +- The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +- The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +- The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +- The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +- The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +- The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +- The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | +
+- The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + - To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + - To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.18/reference/slsa.md b/docs/versioned_docs/version-2.18/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.18/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.18/reference/terraform.md b/docs/versioned_docs/version-2.18/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.18/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.18/workflows/cert-manager.md b/docs/versioned_docs/version-2.18/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.18/workflows/config.md b/docs/versioned_docs/version-2.18/workflows/config.md new file mode 100644 index 000000000..a8a52980e --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.18/workflows/create.md b/docs/versioned_docs/version-2.18/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.18/workflows/lb.md b/docs/versioned_docs/version-2.18/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.18/workflows/recovery.md b/docs/versioned_docs/version-2.18/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.18/workflows/s3proxy.md b/docs/versioned_docs/version-2.18/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.18/workflows/sbom.md b/docs/versioned_docs/version-2.18/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.18/workflows/scale.md b/docs/versioned_docs/version-2.18/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.18/workflows/storage.md b/docs/versioned_docs/version-2.18/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.18/workflows/terraform-provider.md b/docs/versioned_docs/version-2.18/workflows/terraform-provider.md new file mode 100644 index 000000000..ed8f46eda --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/terraform-provider.md @@ -0,0 +1,129 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + When creating a cluster on Azure, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you + can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.18/workflows/troubleshooting.md b/docs/versioned_docs/version-2.18/workflows/troubleshooting.md new file mode 100644 index 000000000..195bce1cc --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/troubleshooting.md @@ -0,0 +1,151 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.18/workflows/trusted-launch.md b/docs/versioned_docs/version-2.18/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.18/workflows/upgrade.md b/docs/versioned_docs/version-2.18/workflows/upgrade.md new file mode 100644 index 000000000..7348c0dbc --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting it's availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.18/workflows/verify-cli.md b/docs/versioned_docs/version-2.18/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.18/workflows/verify-cluster.md b/docs/versioned_docs/version-2.18/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.18/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.19/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.19/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.19/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.19/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.19/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.19/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.19/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.19/_media/concept-constellation.svg b/docs/versioned_docs/version-2.19/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.19/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.19/_media/concept-managed.svg b/docs/versioned_docs/version-2.19/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.19/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.19/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.19/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.19/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.19/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.19/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.19/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.19/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.19/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.19/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.19/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.19/_media/tcb.svg b/docs/versioned_docs/version-2.19/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.19/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.19/architecture/attestation.md b/docs/versioned_docs/version-2.19/architecture/attestation.md new file mode 100644 index 000000000..9bd157460 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/attestation.md @@ -0,0 +1,409 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.19/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.19/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.19/architecture/images.md b/docs/versioned_docs/version-2.19/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.19/architecture/keys.md b/docs/versioned_docs/version-2.19/architecture/keys.md new file mode 100644 index 000000000..553d9d4e2 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/keys.md @@ -0,0 +1,131 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). +Cilium supports [key rotation](https://docs.cilium.io/en/stable/security/network/encryption-ipsec/#key-rotation) for the long-term node keys via Kubernetes secrets. + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.19/architecture/microservices.md b/docs/versioned_docs/version-2.19/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.19/architecture/networking.md b/docs/versioned_docs/version-2.19/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.19/architecture/observability.md b/docs/versioned_docs/version-2.19/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.19/architecture/orchestration.md b/docs/versioned_docs/version-2.19/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.19/architecture/overview.md b/docs/versioned_docs/version-2.19/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.19/architecture/versions.md b/docs/versioned_docs/version-2.19/architecture/versions.md new file mode 100644 index 000000000..9d5a064e0 --- /dev/null +++ b/docs/versioned_docs/version-2.19/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.28.15 +* v1.29.11 +* v1.30.7 diff --git a/docs/versioned_docs/version-2.19/getting-started/examples.md b/docs/versioned_docs/version-2.19/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.19/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.19/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.19/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.19/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.19/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.19/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.19/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.19/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.19/getting-started/first-steps.md b/docs/versioned_docs/version-2.19/getting-started/first-steps.md new file mode 100644 index 000000000..2afe95635 --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/first-steps.md @@ -0,0 +1,235 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + :::caution + + `stackitProjectID` refers to the ID of your STACKIT project. The STACKIT portal also shows the OpenStack ID that's associated with your project in some places. Make sure you insert the STACKIT project ID in the `constellation-conf.yaml` file. It's of the format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. + + ::: + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.19/getting-started/install.md b/docs/versioned_docs/version-2.19/getting-started/install.md new file mode 100644 index 000000000..29be1e7f6 --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/install.md @@ -0,0 +1,439 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file with the credentials from the User Access Token under: + * Linux: `~/.config/openstack/clouds.yaml` + * macOS: `/Users//Library/Application Support/openstack/clouds.yaml` or `/etc/openstack/clouds.yaml` + * Windows: `%AppData%\openstack\clouds.yaml` + + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_STACKIT_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +:::caution + +`project_id` refers to the ID of your OpenStack project. The STACKIT portal also shows the STACKIT ID that's associated with your project in some places. Make sure you insert the OpenStack project ID in the `clouds.yaml` file. + +::: + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.19/getting-started/marketplaces.md b/docs/versioned_docs/version-2.19/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.19/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.19/intro.md b/docs/versioned_docs/version-2.19/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.19/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.19/overview/clouds.md b/docs/versioned_docs/version-2.19/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.19/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.19/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.19/overview/license.md b/docs/versioned_docs/version-2.19/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.19/overview/performance/application.md b/docs/versioned_docs/version-2.19/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.19/overview/performance/compute.md b/docs/versioned_docs/version-2.19/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.19/overview/performance/io.md b/docs/versioned_docs/version-2.19/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.19/overview/performance/performance.md b/docs/versioned_docs/version-2.19/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.19/overview/product.md b/docs/versioned_docs/version-2.19/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.19/overview/security-benefits.md b/docs/versioned_docs/version-2.19/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.19/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.19/reference/cli.md b/docs/versioned_docs/version-2.19/reference/cli.md new file mode 100644 index 000000000..99acef520 --- /dev/null +++ b/docs/versioned_docs/version-2.19/reference/cli.md @@ -0,0 +1,844 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.19/reference/migration.md b/docs/versioned_docs/version-2.19/reference/migration.md new file mode 100644 index 000000000..0252c409f --- /dev/null +++ b/docs/versioned_docs/version-2.19/reference/migration.md @@ -0,0 +1,128 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrations to v2.19.1 + +### Azure + +* During the upgrade, security rules are migrated and the old ones need to be cleaned up manually by the user. The below script shows how to delete them through the Azure CLI: + +```bash +#!/usr/bin/env bash +name="" # the name provided in the config +uid="" # the cluster id can be retrieved via `yq '.infrastructure.uid' constellation-state.yaml` +resource_group="" # the RG can be retrieved via `yq '.provider.azure.resourceGroup' constellation-conf.yaml` + +rules=( + "kubernetes" + "bootstrapper" + "verify" + "recovery" + "join" + "debugd" + "konnectivity" +) + +for rule in "${rules[@]}"; do + echo "Deleting rule: ${rule}" + az network nsg rule delete \ + --resource-group "${resource_group}" \ + --nsg-name "${name}-${uid}" \ + --name "${rule}" +done + +echo "All specified rules have been deleted." +``` + +## Migrations to v2.19.0 + +### Azure + +* To allow seamless upgrades on Azure when Kubernetes services of type `LoadBalancer` are deployed, the target + load balancer in which the `cloud-controller-manager` creates load balancing rules was changed. Instead of using the load balancer + created and maintained by the CLI's Terraform code, the `cloud-controller-manager` now creates its own load balancer in Azure. + If your Constellation has services of type `LoadBalancer`, please remove them before the upgrade and re-apply them + afterward. + +## Migrating from Azure's service principal authentication to managed identity authentication (during the upgrade to Constellation v2.8.0) + +* The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +* To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +* Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +* To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + +## Migrating from CLI versions before 2.10 + +* AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +* The global `nodeGroups` field was added. +* The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +* The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +* The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +* The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +* The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +* The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +* The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +* The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | + +
+* The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + * To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + * To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.19/reference/slsa.md b/docs/versioned_docs/version-2.19/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.19/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.19/reference/terraform.md b/docs/versioned_docs/version-2.19/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.19/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.19/workflows/cert-manager.md b/docs/versioned_docs/version-2.19/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.19/workflows/config.md b/docs/versioned_docs/version-2.19/workflows/config.md new file mode 100644 index 000000000..a8a52980e --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.19/workflows/create.md b/docs/versioned_docs/version-2.19/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.19/workflows/lb.md b/docs/versioned_docs/version-2.19/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.19/workflows/recovery.md b/docs/versioned_docs/version-2.19/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.19/workflows/s3proxy.md b/docs/versioned_docs/version-2.19/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.19/workflows/sbom.md b/docs/versioned_docs/version-2.19/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.19/workflows/scale.md b/docs/versioned_docs/version-2.19/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.19/workflows/storage.md b/docs/versioned_docs/version-2.19/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.19/workflows/terraform-provider.md b/docs/versioned_docs/version-2.19/workflows/terraform-provider.md new file mode 100644 index 000000000..c7a795d3f --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/terraform-provider.md @@ -0,0 +1,140 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +:::info +On SEV-SNP, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + +::: + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.19/workflows/troubleshooting.md b/docs/versioned_docs/version-2.19/workflows/troubleshooting.md new file mode 100644 index 000000000..195bce1cc --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/troubleshooting.md @@ -0,0 +1,151 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.19/workflows/trusted-launch.md b/docs/versioned_docs/version-2.19/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.19/workflows/upgrade.md b/docs/versioned_docs/version-2.19/workflows/upgrade.md new file mode 100644 index 000000000..3db2ecad6 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting its availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.19/workflows/verify-cli.md b/docs/versioned_docs/version-2.19/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.19/workflows/verify-cluster.md b/docs/versioned_docs/version-2.19/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.19/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.2/architecture/attestation.md b/docs/versioned_docs/version-2.2/architecture/attestation.md index c70a61264..c09d0f546 100644 --- a/docs/versioned_docs/version-2.2/architecture/attestation.md +++ b/docs/versioned_docs/version-2.2/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,8 +217,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.2/architecture/keys.md b/docs/versioned_docs/version-2.2/architecture/keys.md index aa4e35496..b7d7ef6f5 100644 --- a/docs/versioned_docs/version-2.2/architecture/keys.md +++ b/docs/versioned_docs/version-2.2/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.2/getting-started/first-steps.md b/docs/versioned_docs/version-2.2/getting-started/first-steps.md index 2850176a0..ef4861cbe 100644 --- a/docs/versioned_docs/version-2.2/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.2/getting-started/first-steps.md @@ -11,36 +11,36 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step 1. Create the configuration file for your selected cloud provider. - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in your current working directory. 2. Fill in your cloud provider specific information. - - + + You need several resources for the cluster. You can use the following `az` script to create them: @@ -71,8 +71,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Run `constellation config instance-types` to get the list of all supported options. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -118,8 +118,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Run `constellation config instance-types` to get the list of all supported options. - - + + You need a service account for the cluster. You can use the following `gcloud` script to create it: @@ -142,18 +142,18 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines from the N2D family with a minimum of 4 vCPUs. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). - * **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. + * **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). - * **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. + * **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -171,8 +171,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Supported are all machines from the N2D family with a minimum of 4 vCPUs. It defaults to `n2d-standard-4` (4 vCPUs, 16 GB RAM), but you can use any other VMs from the same family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -211,8 +211,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - - + + :::info diff --git a/docs/versioned_docs/version-2.2/getting-started/install.md b/docs/versioned_docs/version-2.2/getting-started/install.md index d717dcb34..439b734dd 100644 --- a/docs/versioned_docs/version-2.2/getting-started/install.md +++ b/docs/versioned_docs/version-2.2/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,14 +105,15 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + The following [resource providers need to be need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Compute` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` -* `microsoft.insights` + +- `Microsoft.Compute` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` +- `microsoft.insights` By default, Constellation tries to register these automatically if they haven't been registered before. @@ -127,8 +125,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -140,8 +138,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -272,8 +270,8 @@ such as `PowerUserAccess`, or use the following minimal set of permissions: Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -283,8 +281,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -300,8 +298,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -324,8 +322,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -341,10 +339,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.2/overview/clouds.md b/docs/versioned_docs/version-2.2/overview/clouds.md index 01e7a00c5..c48f23cf0 100644 --- a/docs/versioned_docs/version-2.2/overview/clouds.md +++ b/docs/versioned_docs/version-2.2/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.2/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.2/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.2/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.2/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.2/overview/product.md b/docs/versioned_docs/version-2.2/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.2/overview/product.md +++ b/docs/versioned_docs/version-2.2/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.2/workflows/create.md b/docs/versioned_docs/version-2.2/workflows/create.md index d886fb347..dcb3ff285 100644 --- a/docs/versioned_docs/version-2.2/workflows/create.md +++ b/docs/versioned_docs/version-2.2/workflows/create.md @@ -19,29 +19,29 @@ This step creates the necessary resources for your cluster in your cloud environ Generate a configuration file for your cloud service provider (CSP): - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. [Fill in your CSP-specific information](../getting-started/first-steps.md#create-a-cluster) before you continue. diff --git a/docs/versioned_docs/version-2.2/workflows/recovery.md b/docs/versioned_docs/version-2.2/workflows/recovery.md index fd610fc67..0fd171036 100644 --- a/docs/versioned_docs/version-2.2/workflows/recovery.md +++ b/docs/versioned_docs/version-2.2/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.2/workflows/sbom.md b/docs/versioned_docs/version-2.2/workflows/sbom.md index ec9834b4f..e8ba25a64 100644 --- a/docs/versioned_docs/version-2.2/workflows/sbom.md +++ b/docs/versioned_docs/version-2.2/workflows/sbom.md @@ -15,7 +15,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -36,7 +36,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.2/workflows/scale.md b/docs/versioned_docs/version-2.2/workflows/scale.md index 3b7c0d479..bce045c66 100644 --- a/docs/versioned_docs/version-2.2/workflows/scale.md +++ b/docs/versioned_docs/version-2.2/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + :::caution @@ -72,8 +72,8 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + ## Control-plane node scaling @@ -81,24 +81,24 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + :::caution @@ -106,7 +106,7 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.2/workflows/storage.md b/docs/versioned_docs/version-2.2/workflows/storage.md index 878449fa5..c322d97d1 100644 --- a/docs/versioned_docs/version-2.2/workflows/storage.md +++ b/docs/versioned_docs/version-2.2/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -56,8 +56,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The following installation guide gives an overview of how to securely use CSI-based cloud storage for persistent volumes in Constellation. - - + + 1. Install the driver: @@ -67,8 +67,8 @@ The following installation guide gives an overview of how to securely use CSI-ba helm install azuredisk-csi-driver charts/edgeless --namespace kube-system ``` - - + + 1. Install the driver: @@ -77,8 +77,8 @@ The following installation guide gives an overview of how to securely use CSI-ba helm install gcp-compute-persistent-disk-csi-driver charts/ --namespace kube-system ``` - - + + :::caution @@ -88,8 +88,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + :::info @@ -160,8 +160,8 @@ The default storage class is responsible for all persistent volume claims that d The previous instructions create a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -207,8 +207,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -254,8 +254,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -265,5 +265,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.2/workflows/troubleshooting.md b/docs/versioned_docs/version-2.2/workflows/troubleshooting.md index ad5e1c51b..59015efcb 100644 --- a/docs/versioned_docs/version-2.2/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.2/workflows/troubleshooting.md @@ -5,6 +5,7 @@ This section aids you in finding problems when working with Constellation. ## Azure: Resource Providers can't be registered On Azure, you may receive the following error when running `create` or `terminate` with limited IAM permissions: + ```shell-session Error: Error ensuring Resource Providers are registered. @@ -21,11 +22,13 @@ To continue, please ensure that the [required resource providers](../getting-sta Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `create` or `terminate` again. For example: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation create --control-plane-nodes 1 --worker-nodes 2 -y ``` Or alternatively, for `terminate`: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate ``` @@ -36,8 +39,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -47,8 +50,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -63,16 +66,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ## Connect to nodes via SSH diff --git a/docs/versioned_docs/version-2.2/workflows/trusted-launch.md b/docs/versioned_docs/version-2.2/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.2/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.2/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.2/workflows/verify-cli.md b/docs/versioned_docs/version-2.2/workflows/verify-cli.md index 0a52fedd4..52ed24d95 100644 --- a/docs/versioned_docs/version-2.2/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.2/workflows/verify-cli.md @@ -1,6 +1,6 @@ # Verify the CLI -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -12,7 +12,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -25,7 +25,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.20/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.20/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.20/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.20/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.20/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.20/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.20/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.20/_media/concept-constellation.svg b/docs/versioned_docs/version-2.20/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.20/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.20/_media/concept-managed.svg b/docs/versioned_docs/version-2.20/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.20/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.20/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.20/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.20/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.20/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.20/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.20/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.20/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.20/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.20/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.20/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.20/_media/tcb.svg b/docs/versioned_docs/version-2.20/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.20/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.20/architecture/attestation.md b/docs/versioned_docs/version-2.20/architecture/attestation.md new file mode 100644 index 000000000..9bd157460 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/attestation.md @@ -0,0 +1,409 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.20/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.20/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.20/architecture/images.md b/docs/versioned_docs/version-2.20/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.20/architecture/keys.md b/docs/versioned_docs/version-2.20/architecture/keys.md new file mode 100644 index 000000000..49821cd0b --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/keys.md @@ -0,0 +1,130 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.20/architecture/microservices.md b/docs/versioned_docs/version-2.20/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.20/architecture/networking.md b/docs/versioned_docs/version-2.20/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.20/architecture/observability.md b/docs/versioned_docs/version-2.20/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.20/architecture/orchestration.md b/docs/versioned_docs/version-2.20/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.20/architecture/overview.md b/docs/versioned_docs/version-2.20/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.20/architecture/versions.md b/docs/versioned_docs/version-2.20/architecture/versions.md new file mode 100644 index 000000000..9d5a064e0 --- /dev/null +++ b/docs/versioned_docs/version-2.20/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.28.15 +* v1.29.11 +* v1.30.7 diff --git a/docs/versioned_docs/version-2.20/getting-started/examples.md b/docs/versioned_docs/version-2.20/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.20/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.20/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.20/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.20/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.20/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.20/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.20/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.20/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.20/getting-started/first-steps.md b/docs/versioned_docs/version-2.20/getting-started/first-steps.md new file mode 100644 index 000000000..2afe95635 --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/first-steps.md @@ -0,0 +1,235 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + :::caution + + `stackitProjectID` refers to the ID of your STACKIT project. The STACKIT portal also shows the OpenStack ID that's associated with your project in some places. Make sure you insert the STACKIT project ID in the `constellation-conf.yaml` file. It's of the format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. + + ::: + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.20/getting-started/install.md b/docs/versioned_docs/version-2.20/getting-started/install.md new file mode 100644 index 000000000..29be1e7f6 --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/install.md @@ -0,0 +1,439 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file with the credentials from the User Access Token under: + * Linux: `~/.config/openstack/clouds.yaml` + * macOS: `/Users//Library/Application Support/openstack/clouds.yaml` or `/etc/openstack/clouds.yaml` + * Windows: `%AppData%\openstack\clouds.yaml` + + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_STACKIT_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +:::caution + +`project_id` refers to the ID of your OpenStack project. The STACKIT portal also shows the STACKIT ID that's associated with your project in some places. Make sure you insert the OpenStack project ID in the `clouds.yaml` file. + +::: + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.20/getting-started/marketplaces.md b/docs/versioned_docs/version-2.20/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.20/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.20/intro.md b/docs/versioned_docs/version-2.20/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.20/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.20/overview/clouds.md b/docs/versioned_docs/version-2.20/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.20/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.20/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.20/overview/license.md b/docs/versioned_docs/version-2.20/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.20/overview/performance/application.md b/docs/versioned_docs/version-2.20/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.20/overview/performance/compute.md b/docs/versioned_docs/version-2.20/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.20/overview/performance/io.md b/docs/versioned_docs/version-2.20/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.20/overview/performance/performance.md b/docs/versioned_docs/version-2.20/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.20/overview/product.md b/docs/versioned_docs/version-2.20/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.20/overview/security-benefits.md b/docs/versioned_docs/version-2.20/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.20/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.20/reference/cli.md b/docs/versioned_docs/version-2.20/reference/cli.md new file mode 100644 index 000000000..99acef520 --- /dev/null +++ b/docs/versioned_docs/version-2.20/reference/cli.md @@ -0,0 +1,844 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.20/reference/migration.md b/docs/versioned_docs/version-2.20/reference/migration.md new file mode 100644 index 000000000..0252c409f --- /dev/null +++ b/docs/versioned_docs/version-2.20/reference/migration.md @@ -0,0 +1,128 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrations to v2.19.1 + +### Azure + +* During the upgrade, security rules are migrated and the old ones need to be cleaned up manually by the user. The below script shows how to delete them through the Azure CLI: + +```bash +#!/usr/bin/env bash +name="" # the name provided in the config +uid="" # the cluster id can be retrieved via `yq '.infrastructure.uid' constellation-state.yaml` +resource_group="" # the RG can be retrieved via `yq '.provider.azure.resourceGroup' constellation-conf.yaml` + +rules=( + "kubernetes" + "bootstrapper" + "verify" + "recovery" + "join" + "debugd" + "konnectivity" +) + +for rule in "${rules[@]}"; do + echo "Deleting rule: ${rule}" + az network nsg rule delete \ + --resource-group "${resource_group}" \ + --nsg-name "${name}-${uid}" \ + --name "${rule}" +done + +echo "All specified rules have been deleted." +``` + +## Migrations to v2.19.0 + +### Azure + +* To allow seamless upgrades on Azure when Kubernetes services of type `LoadBalancer` are deployed, the target + load balancer in which the `cloud-controller-manager` creates load balancing rules was changed. Instead of using the load balancer + created and maintained by the CLI's Terraform code, the `cloud-controller-manager` now creates its own load balancer in Azure. + If your Constellation has services of type `LoadBalancer`, please remove them before the upgrade and re-apply them + afterward. + +## Migrating from Azure's service principal authentication to managed identity authentication (during the upgrade to Constellation v2.8.0) + +* The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +* To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +* Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +* To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + +## Migrating from CLI versions before 2.10 + +* AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +* The global `nodeGroups` field was added. +* The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +* The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +* The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +* The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +* The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +* The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +* The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +* The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | + +
+* The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + * To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + * To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.20/reference/slsa.md b/docs/versioned_docs/version-2.20/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.20/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.20/reference/terraform.md b/docs/versioned_docs/version-2.20/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.20/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.20/workflows/cert-manager.md b/docs/versioned_docs/version-2.20/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.20/workflows/config.md b/docs/versioned_docs/version-2.20/workflows/config.md new file mode 100644 index 000000000..a8a52980e --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.20/workflows/create.md b/docs/versioned_docs/version-2.20/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.20/workflows/lb.md b/docs/versioned_docs/version-2.20/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.20/workflows/recovery.md b/docs/versioned_docs/version-2.20/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.20/workflows/reproducible-builds.md b/docs/versioned_docs/version-2.20/workflows/reproducible-builds.md new file mode 100644 index 000000000..e3bc46095 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/reproducible-builds.md @@ -0,0 +1,63 @@ +# Reproduce released artifacts + +Constellation has first-class support for [reproducible builds](https://reproducible-builds.org). +Reproducing the released artifacts is an alternative to [signature verification](verify-cli.md) that doesn't require trusting Edgeless Systems' release process. +The following sections describe how to rebuild an artifact and how Constellation ensures that the rebuild reproduces the artifacts bit-by-bit. + +## Build environment prerequisites + +The build systems used by Constellation - [Bazel](https://bazel.build/) and [Nix](https://nixos.org) - are designed for deterministic, reproducible builds. +These two dependencies should be the only prerequisites for a successful build. +However, it can't be ruled out completely that peculiarities of the host affect the build result. +Thus, we recommend the following host setup for best results: + +1. A Linux operating system not older than v5.4. +2. The GNU C library not older than v2.31 (avoid `musl`). +3. GNU `coreutils` not older than v8.30 (avoid `busybox`). +4. An `ext4` filesystem for building. +5. AppArmor turned off. + +This is given, for example, on an Ubuntu 22.04 system, which is also used for reproducibility tests. + +:::note + +To avoid any backwards-compatibility issues, the host software versions should also not be much newer than the Constellation release. + +::: + +## Run the build + +The following instructions outline qualitatively how to reproduce a build. +Constellation implements these instructions in the [Reproducible Builds workflow](https://github.com/edgelesssys/constellation/actions/workflows/reproducible-builds.yml), which continuously tests for reproducibility. +The workflow is a good place to look up specific version numbers and build steps. + +1. Check out the Constellation repository at the tag corresponding to the release. + + ```bash + git clone https://github.com/edgelesssys/constellation.git + cd constellation + git checkout v2.20.0 + ``` + +2. [Install the Bazel release](https://bazel.build/install) specified in `.bazelversion`. +3. [Install Nix](https://nixos.org/download/) (any recent version should do). +4. Run the build with `bazel build $target` for one of the following targets of interest: + + ```data + //cli:cli_enterprise_darwin_amd64 + //cli:cli_enterprise_darwin_arm64 + //cli:cli_enterprise_linux_amd64 + //cli:cli_enterprise_linux_arm64 + //cli:cli_enterprise_windows_amd64 + ``` + +5. Compare the build result with the downloaded release artifact. + + + +## Feedback + +Reproduction failures often indicate a bug in the build system or in the build definitions. +Therefore, we're interested in any reproducibility issues you might encounter. +[Start a bug report](https://github.com/edgelesssys/constellation/issues/new/choose) and describe the details of your build environment. +Make sure to include your result binary or a [`diffoscope`](https://diffoscope.org/) report, if possible. diff --git a/docs/versioned_docs/version-2.20/workflows/s3proxy.md b/docs/versioned_docs/version-2.20/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.20/workflows/sbom.md b/docs/versioned_docs/version-2.20/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.20/workflows/scale.md b/docs/versioned_docs/version-2.20/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.20/workflows/storage.md b/docs/versioned_docs/version-2.20/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.20/workflows/terraform-provider.md b/docs/versioned_docs/version-2.20/workflows/terraform-provider.md new file mode 100644 index 000000000..c7a795d3f --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/terraform-provider.md @@ -0,0 +1,140 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +:::info +On SEV-SNP, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + +::: + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.20/workflows/troubleshooting.md b/docs/versioned_docs/version-2.20/workflows/troubleshooting.md new file mode 100644 index 000000000..195bce1cc --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/troubleshooting.md @@ -0,0 +1,151 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.20/workflows/trusted-launch.md b/docs/versioned_docs/version-2.20/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.20/workflows/upgrade.md b/docs/versioned_docs/version-2.20/workflows/upgrade.md new file mode 100644 index 000000000..3db2ecad6 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting its availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.20/workflows/verify-cli.md b/docs/versioned_docs/version-2.20/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.20/workflows/verify-cluster.md b/docs/versioned_docs/version-2.20/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.20/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.21/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.21/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.21/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.21/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.21/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.21/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.21/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.21/_media/concept-constellation.svg b/docs/versioned_docs/version-2.21/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.21/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.21/_media/concept-managed.svg b/docs/versioned_docs/version-2.21/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.21/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.21/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.21/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.21/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.21/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.21/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.21/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.21/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.21/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.21/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.21/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.21/_media/tcb.svg b/docs/versioned_docs/version-2.21/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.21/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.21/architecture/attestation.md b/docs/versioned_docs/version-2.21/architecture/attestation.md new file mode 100644 index 000000000..9bd157460 --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/attestation.md @@ -0,0 +1,409 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.21/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.21/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.21/architecture/images.md b/docs/versioned_docs/version-2.21/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.21/architecture/keys.md b/docs/versioned_docs/version-2.21/architecture/keys.md new file mode 100644 index 000000000..49821cd0b --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/keys.md @@ -0,0 +1,130 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.21/architecture/microservices.md b/docs/versioned_docs/version-2.21/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.21/architecture/networking.md b/docs/versioned_docs/version-2.21/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.21/architecture/observability.md b/docs/versioned_docs/version-2.21/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.21/architecture/orchestration.md b/docs/versioned_docs/version-2.21/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.21/architecture/overview.md b/docs/versioned_docs/version-2.21/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.21/architecture/versions.md b/docs/versioned_docs/version-2.21/architecture/versions.md new file mode 100644 index 000000000..9acc866ed --- /dev/null +++ b/docs/versioned_docs/version-2.21/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.29.14 +* v1.30.10 +* v1.31.6 diff --git a/docs/versioned_docs/version-2.21/getting-started/examples.md b/docs/versioned_docs/version-2.21/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.21/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.21/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.21/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.21/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.21/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.21/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.21/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.21/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.21/getting-started/first-steps.md b/docs/versioned_docs/version-2.21/getting-started/first-steps.md new file mode 100644 index 000000000..2afe95635 --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/first-steps.md @@ -0,0 +1,235 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + :::caution + + `stackitProjectID` refers to the ID of your STACKIT project. The STACKIT portal also shows the OpenStack ID that's associated with your project in some places. Make sure you insert the STACKIT project ID in the `constellation-conf.yaml` file. It's of the format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. + + ::: + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.21/getting-started/install.md b/docs/versioned_docs/version-2.21/getting-started/install.md new file mode 100644 index 000000000..f7b36770a --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/install.md @@ -0,0 +1,439 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file with the credentials from the User Access Token under: + * Linux: `~/.config/openstack/clouds.yaml` + * macOS: `/Users//Library/Application Support/openstack/clouds.yaml` or `/etc/openstack/clouds.yaml` + * Windows: `%AppData%\openstack\clouds.yaml` + + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_OPENSTACK_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +:::caution + +`project_id` refers to the ID of your OpenStack project. The STACKIT portal also shows the STACKIT ID that's associated with your project in some places. Make sure you insert the OpenStack project ID in the `clouds.yaml` file. + +::: + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.21/getting-started/marketplaces.md b/docs/versioned_docs/version-2.21/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.21/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.21/intro.md b/docs/versioned_docs/version-2.21/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.21/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.21/overview/clouds.md b/docs/versioned_docs/version-2.21/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.21/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.21/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.21/overview/license.md b/docs/versioned_docs/version-2.21/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.21/overview/performance/application.md b/docs/versioned_docs/version-2.21/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.21/overview/performance/compute.md b/docs/versioned_docs/version-2.21/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.21/overview/performance/io.md b/docs/versioned_docs/version-2.21/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.21/overview/performance/performance.md b/docs/versioned_docs/version-2.21/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.21/overview/product.md b/docs/versioned_docs/version-2.21/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.21/overview/security-benefits.md b/docs/versioned_docs/version-2.21/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.21/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.21/reference/cli.md b/docs/versioned_docs/version-2.21/reference/cli.md new file mode 100644 index 000000000..7bb4d5b40 --- /dev/null +++ b/docs/versioned_docs/version-2.21/reference/cli.md @@ -0,0 +1,873 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster +* [ssh](#constellation-ssh): Prepare your cluster for emergency ssh access + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.30") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --serviceAccountID string ID for the service account that will be created (required) + Must be 6 to 30 lowercase letters, digits, or hyphens. + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation ssh + +Prepare your cluster for emergency ssh access + +### Synopsis + +Prepare your cluster for emergency ssh access and sign a given key pair for authorization. + +``` +constellation ssh [flags] +``` + +### Options + +``` + -h, --help help for ssh + --key string the path to an existing ssh public key +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.21/reference/migration.md b/docs/versioned_docs/version-2.21/reference/migration.md new file mode 100644 index 000000000..0252c409f --- /dev/null +++ b/docs/versioned_docs/version-2.21/reference/migration.md @@ -0,0 +1,128 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrations to v2.19.1 + +### Azure + +* During the upgrade, security rules are migrated and the old ones need to be cleaned up manually by the user. The below script shows how to delete them through the Azure CLI: + +```bash +#!/usr/bin/env bash +name="" # the name provided in the config +uid="" # the cluster id can be retrieved via `yq '.infrastructure.uid' constellation-state.yaml` +resource_group="" # the RG can be retrieved via `yq '.provider.azure.resourceGroup' constellation-conf.yaml` + +rules=( + "kubernetes" + "bootstrapper" + "verify" + "recovery" + "join" + "debugd" + "konnectivity" +) + +for rule in "${rules[@]}"; do + echo "Deleting rule: ${rule}" + az network nsg rule delete \ + --resource-group "${resource_group}" \ + --nsg-name "${name}-${uid}" \ + --name "${rule}" +done + +echo "All specified rules have been deleted." +``` + +## Migrations to v2.19.0 + +### Azure + +* To allow seamless upgrades on Azure when Kubernetes services of type `LoadBalancer` are deployed, the target + load balancer in which the `cloud-controller-manager` creates load balancing rules was changed. Instead of using the load balancer + created and maintained by the CLI's Terraform code, the `cloud-controller-manager` now creates its own load balancer in Azure. + If your Constellation has services of type `LoadBalancer`, please remove them before the upgrade and re-apply them + afterward. + +## Migrating from Azure's service principal authentication to managed identity authentication (during the upgrade to Constellation v2.8.0) + +* The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +* To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +* Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +* To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + +## Migrating from CLI versions before 2.10 + +* AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +* The global `nodeGroups` field was added. +* The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +* The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +* The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +* The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +* The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +* The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +* The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +* The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | + +
+* The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + * To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + * To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.21/reference/slsa.md b/docs/versioned_docs/version-2.21/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.21/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.21/reference/terraform.md b/docs/versioned_docs/version-2.21/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.21/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.21/workflows/cert-manager.md b/docs/versioned_docs/version-2.21/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.21/workflows/config.md b/docs/versioned_docs/version-2.21/workflows/config.md new file mode 100644 index 000000000..a8a52980e --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.21/workflows/create.md b/docs/versioned_docs/version-2.21/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.21/workflows/lb.md b/docs/versioned_docs/version-2.21/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.21/workflows/recovery.md b/docs/versioned_docs/version-2.21/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.21/workflows/reproducible-builds.md b/docs/versioned_docs/version-2.21/workflows/reproducible-builds.md new file mode 100644 index 000000000..e3bc46095 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/reproducible-builds.md @@ -0,0 +1,63 @@ +# Reproduce released artifacts + +Constellation has first-class support for [reproducible builds](https://reproducible-builds.org). +Reproducing the released artifacts is an alternative to [signature verification](verify-cli.md) that doesn't require trusting Edgeless Systems' release process. +The following sections describe how to rebuild an artifact and how Constellation ensures that the rebuild reproduces the artifacts bit-by-bit. + +## Build environment prerequisites + +The build systems used by Constellation - [Bazel](https://bazel.build/) and [Nix](https://nixos.org) - are designed for deterministic, reproducible builds. +These two dependencies should be the only prerequisites for a successful build. +However, it can't be ruled out completely that peculiarities of the host affect the build result. +Thus, we recommend the following host setup for best results: + +1. A Linux operating system not older than v5.4. +2. The GNU C library not older than v2.31 (avoid `musl`). +3. GNU `coreutils` not older than v8.30 (avoid `busybox`). +4. An `ext4` filesystem for building. +5. AppArmor turned off. + +This is given, for example, on an Ubuntu 22.04 system, which is also used for reproducibility tests. + +:::note + +To avoid any backwards-compatibility issues, the host software versions should also not be much newer than the Constellation release. + +::: + +## Run the build + +The following instructions outline qualitatively how to reproduce a build. +Constellation implements these instructions in the [Reproducible Builds workflow](https://github.com/edgelesssys/constellation/actions/workflows/reproducible-builds.yml), which continuously tests for reproducibility. +The workflow is a good place to look up specific version numbers and build steps. + +1. Check out the Constellation repository at the tag corresponding to the release. + + ```bash + git clone https://github.com/edgelesssys/constellation.git + cd constellation + git checkout v2.20.0 + ``` + +2. [Install the Bazel release](https://bazel.build/install) specified in `.bazelversion`. +3. [Install Nix](https://nixos.org/download/) (any recent version should do). +4. Run the build with `bazel build $target` for one of the following targets of interest: + + ```data + //cli:cli_enterprise_darwin_amd64 + //cli:cli_enterprise_darwin_arm64 + //cli:cli_enterprise_linux_amd64 + //cli:cli_enterprise_linux_arm64 + //cli:cli_enterprise_windows_amd64 + ``` + +5. Compare the build result with the downloaded release artifact. + + + +## Feedback + +Reproduction failures often indicate a bug in the build system or in the build definitions. +Therefore, we're interested in any reproducibility issues you might encounter. +[Start a bug report](https://github.com/edgelesssys/constellation/issues/new/choose) and describe the details of your build environment. +Make sure to include your result binary or a [`diffoscope`](https://diffoscope.org/) report, if possible. diff --git a/docs/versioned_docs/version-2.21/workflows/s3proxy.md b/docs/versioned_docs/version-2.21/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.21/workflows/sbom.md b/docs/versioned_docs/version-2.21/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.21/workflows/scale.md b/docs/versioned_docs/version-2.21/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.21/workflows/storage.md b/docs/versioned_docs/version-2.21/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.21/workflows/terraform-provider.md b/docs/versioned_docs/version-2.21/workflows/terraform-provider.md new file mode 100644 index 000000000..c7a795d3f --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/terraform-provider.md @@ -0,0 +1,140 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +:::info +On SEV-SNP, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + +::: + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.21/workflows/troubleshooting.md b/docs/versioned_docs/version-2.21/workflows/troubleshooting.md new file mode 100644 index 000000000..195bce1cc --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/troubleshooting.md @@ -0,0 +1,151 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` diff --git a/docs/versioned_docs/version-2.21/workflows/trusted-launch.md b/docs/versioned_docs/version-2.21/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.21/workflows/upgrade.md b/docs/versioned_docs/version-2.21/workflows/upgrade.md new file mode 100644 index 000000000..3db2ecad6 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting its availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.21/workflows/verify-cli.md b/docs/versioned_docs/version-2.21/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.21/workflows/verify-cluster.md b/docs/versioned_docs/version-2.21/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.21/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.22/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.22/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.22/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.22/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.22/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.22/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.22/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.22/_media/concept-constellation.svg b/docs/versioned_docs/version-2.22/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.22/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.22/_media/concept-managed.svg b/docs/versioned_docs/version-2.22/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.22/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.22/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.22/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.22/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.22/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.22/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.22/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.22/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.22/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.22/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.22/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.22/_media/tcb.svg b/docs/versioned_docs/version-2.22/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.22/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.22/architecture/attestation.md b/docs/versioned_docs/version-2.22/architecture/attestation.md new file mode 100644 index 000000000..9bd157460 --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/attestation.md @@ -0,0 +1,409 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.22/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.22/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.22/architecture/images.md b/docs/versioned_docs/version-2.22/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.22/architecture/keys.md b/docs/versioned_docs/version-2.22/architecture/keys.md new file mode 100644 index 000000000..49821cd0b --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/keys.md @@ -0,0 +1,130 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.22/architecture/microservices.md b/docs/versioned_docs/version-2.22/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.22/architecture/networking.md b/docs/versioned_docs/version-2.22/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.22/architecture/observability.md b/docs/versioned_docs/version-2.22/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.22/architecture/orchestration.md b/docs/versioned_docs/version-2.22/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.22/architecture/overview.md b/docs/versioned_docs/version-2.22/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.22/architecture/versions.md b/docs/versioned_docs/version-2.22/architecture/versions.md new file mode 100644 index 000000000..6f06c011b --- /dev/null +++ b/docs/versioned_docs/version-2.22/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.29.15 +* v1.30.11 +* v1.31.7 diff --git a/docs/versioned_docs/version-2.22/getting-started/examples.md b/docs/versioned_docs/version-2.22/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.22/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.22/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.22/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.22/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.22/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.22/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.22/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.22/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.22/getting-started/first-steps.md b/docs/versioned_docs/version-2.22/getting-started/first-steps.md new file mode 100644 index 000000000..fb8437a06 --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/first-steps.md @@ -0,0 +1,235 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --prefix=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + :::caution + + `stackitProjectID` refers to the ID of your STACKIT project. The STACKIT portal also shows the OpenStack ID that's associated with your project in some places. Make sure you insert the STACKIT project ID in the `constellation-conf.yaml` file. It's of the format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. + + ::: + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.22/getting-started/install.md b/docs/versioned_docs/version-2.22/getting-started/install.md new file mode 100644 index 000000000..f120b865a --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/install.md @@ -0,0 +1,442 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.roles.create` +* `iam.roles.delete` +* `iam.roles.get` +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file with the credentials from the User Access Token under: + * Linux: `~/.config/openstack/clouds.yaml` + * macOS: `/Users//Library/Application Support/openstack/clouds.yaml` or `/etc/openstack/clouds.yaml` + * Windows: `%AppData%\openstack\clouds.yaml` + + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_OPENSTACK_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +:::caution + +`project_id` refers to the ID of your OpenStack project. The STACKIT portal also shows the STACKIT ID that's associated with your project in some places. Make sure you insert the OpenStack project ID in the `clouds.yaml` file. + +::: + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.22/getting-started/marketplaces.md b/docs/versioned_docs/version-2.22/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.22/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.22/intro.md b/docs/versioned_docs/version-2.22/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.22/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.22/overview/clouds.md b/docs/versioned_docs/version-2.22/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.22/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.22/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.22/overview/license.md b/docs/versioned_docs/version-2.22/overview/license.md new file mode 100644 index 000000000..34122c025 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/license.md @@ -0,0 +1,33 @@ +# License + +## Source code + +Constellation's source code is available on [GitHub](https://github.com/edgelesssys/constellation) under the [GNU Affero General Public License v3.0](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +## Binaries + +Edgeless Systems provides ready-to-use and [signed](../architecture/attestation.md#chain-of-trust) binaries of Constellation. This includes the CLI and the [node images](../architecture/images.md). + +These binaries may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +The Constellation CLI displays relevant license information when you initialize your cluster. You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Terraform provider + +Edgeless Systems provides a [Terraform provider](https://github.com/edgelesssys/terraform-provider-constellation/releases), which may be used free of charge within the bounds of Constellation's [**Community License**](#community-license). An [**Enterprise License**](#enterprise-license) can be purchased from Edgeless Systems. + +You are responsible for staying within the bounds of your respective license. Constellation doesn't enforce any limits so as not to endanger your cluster's availability. + +## Community License + +You are free to use the Constellation binaries provided by Edgeless Systems to create services for internal consumption, evaluation purposes, or non-commercial use. You must not use the Constellation binaries to provide commercial hosted services to third parties. Edgeless Systems gives no warranties and offers no support. + +## Enterprise License + +Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.22/overview/performance/application.md b/docs/versioned_docs/version-2.22/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.22/overview/performance/compute.md b/docs/versioned_docs/version-2.22/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.22/overview/performance/io.md b/docs/versioned_docs/version-2.22/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.22/overview/performance/performance.md b/docs/versioned_docs/version-2.22/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.22/overview/product.md b/docs/versioned_docs/version-2.22/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.22/overview/security-benefits.md b/docs/versioned_docs/version-2.22/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.22/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.22/reference/cli.md b/docs/versioned_docs/version-2.22/reference/cli.md new file mode 100644 index 000000000..7cbc0be8d --- /dev/null +++ b/docs/versioned_docs/version-2.22/reference/cli.md @@ -0,0 +1,873 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster +* [ssh](#constellation-ssh): Generate a certificate for emergency SSH access + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.30") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --prefix string Prefix for the service account ID and VM ID that will be created (required) + Must be letters, digits, or hyphens. + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation ssh + +Generate a certificate for emergency SSH access + +### Synopsis + +Generate a certificate for emergency SSH access to your SSH-enabled constellation cluster. + +``` +constellation ssh [flags] +``` + +### Options + +``` + -h, --help help for ssh + --key string the path to an existing SSH public key +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.22/reference/migration.md b/docs/versioned_docs/version-2.22/reference/migration.md new file mode 100644 index 000000000..36bfb1462 --- /dev/null +++ b/docs/versioned_docs/version-2.22/reference/migration.md @@ -0,0 +1,134 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrations to v2.19.1 + +### Azure + +* During the upgrade, security rules are migrated and the old ones need to be cleaned up manually by the user. The below script shows how to delete them through the Azure CLI: + +```bash +#!/usr/bin/env bash +name="" # the name provided in the config +uid="" # the cluster id can be retrieved via `yq '.infrastructure.uid' constellation-state.yaml` +resource_group="" # the RG can be retrieved via `yq '.provider.azure.resourceGroup' constellation-conf.yaml` + +rules=( + "kubernetes" + "bootstrapper" + "verify" + "recovery" + "join" + "debugd" + "konnectivity" +) + +for rule in "${rules[@]}"; do + echo "Deleting rule: ${rule}" + az network nsg rule delete \ + --resource-group "${resource_group}" \ + --nsg-name "${name}-${uid}" \ + --name "${rule}" +done + +echo "All specified rules have been deleted." +``` + +## Migrating from CLI versions before 2.21.1 + +### AWS + +* AWS clusters that use `LoadBalancer` resources require more IAM permissions. Please upgrade your IAM roles using `constellation iam upgrade apply`. This will show necessary changes and apply them, if desired. + +## Migrating from CLI versions before 2.19.0 + +### Azure + +* To allow seamless upgrades on Azure when Kubernetes services of type `LoadBalancer` are deployed, the target + load balancer in which the `cloud-controller-manager` creates load balancing rules was changed. Instead of using the load balancer + created and maintained by the CLI's Terraform code, the `cloud-controller-manager` now creates its own load balancer in Azure. + If your Constellation has services of type `LoadBalancer`, please remove them before the upgrade and re-apply them + afterward. + +## Migrating from CLI versions before 2.18.0 + +* The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +* To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +* Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +* To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + +## Migrating from CLI versions before 2.10 + +* AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +* The global `nodeGroups` field was added. +* The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +* The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +* The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +* The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +* The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +* The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +* The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +* The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | + +
+* The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + * To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + * To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.22/reference/slsa.md b/docs/versioned_docs/version-2.22/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.22/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.22/reference/terraform.md b/docs/versioned_docs/version-2.22/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.22/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.22/workflows/cert-manager.md b/docs/versioned_docs/version-2.22/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.22/workflows/config.md b/docs/versioned_docs/version-2.22/workflows/config.md new file mode 100644 index 000000000..7868ff1be --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --prefix=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.22/workflows/create.md b/docs/versioned_docs/version-2.22/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.22/workflows/lb.md b/docs/versioned_docs/version-2.22/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.22/workflows/recovery.md b/docs/versioned_docs/version-2.22/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.22/workflows/reproducible-builds.md b/docs/versioned_docs/version-2.22/workflows/reproducible-builds.md new file mode 100644 index 000000000..e3bc46095 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/reproducible-builds.md @@ -0,0 +1,63 @@ +# Reproduce released artifacts + +Constellation has first-class support for [reproducible builds](https://reproducible-builds.org). +Reproducing the released artifacts is an alternative to [signature verification](verify-cli.md) that doesn't require trusting Edgeless Systems' release process. +The following sections describe how to rebuild an artifact and how Constellation ensures that the rebuild reproduces the artifacts bit-by-bit. + +## Build environment prerequisites + +The build systems used by Constellation - [Bazel](https://bazel.build/) and [Nix](https://nixos.org) - are designed for deterministic, reproducible builds. +These two dependencies should be the only prerequisites for a successful build. +However, it can't be ruled out completely that peculiarities of the host affect the build result. +Thus, we recommend the following host setup for best results: + +1. A Linux operating system not older than v5.4. +2. The GNU C library not older than v2.31 (avoid `musl`). +3. GNU `coreutils` not older than v8.30 (avoid `busybox`). +4. An `ext4` filesystem for building. +5. AppArmor turned off. + +This is given, for example, on an Ubuntu 22.04 system, which is also used for reproducibility tests. + +:::note + +To avoid any backwards-compatibility issues, the host software versions should also not be much newer than the Constellation release. + +::: + +## Run the build + +The following instructions outline qualitatively how to reproduce a build. +Constellation implements these instructions in the [Reproducible Builds workflow](https://github.com/edgelesssys/constellation/actions/workflows/reproducible-builds.yml), which continuously tests for reproducibility. +The workflow is a good place to look up specific version numbers and build steps. + +1. Check out the Constellation repository at the tag corresponding to the release. + + ```bash + git clone https://github.com/edgelesssys/constellation.git + cd constellation + git checkout v2.20.0 + ``` + +2. [Install the Bazel release](https://bazel.build/install) specified in `.bazelversion`. +3. [Install Nix](https://nixos.org/download/) (any recent version should do). +4. Run the build with `bazel build $target` for one of the following targets of interest: + + ```data + //cli:cli_enterprise_darwin_amd64 + //cli:cli_enterprise_darwin_arm64 + //cli:cli_enterprise_linux_amd64 + //cli:cli_enterprise_linux_arm64 + //cli:cli_enterprise_windows_amd64 + ``` + +5. Compare the build result with the downloaded release artifact. + + + +## Feedback + +Reproduction failures often indicate a bug in the build system or in the build definitions. +Therefore, we're interested in any reproducibility issues you might encounter. +[Start a bug report](https://github.com/edgelesssys/constellation/issues/new/choose) and describe the details of your build environment. +Make sure to include your result binary or a [`diffoscope`](https://diffoscope.org/) report, if possible. diff --git a/docs/versioned_docs/version-2.22/workflows/s3proxy.md b/docs/versioned_docs/version-2.22/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.22/workflows/sbom.md b/docs/versioned_docs/version-2.22/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.22/workflows/scale.md b/docs/versioned_docs/version-2.22/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.22/workflows/storage.md b/docs/versioned_docs/version-2.22/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.22/workflows/terraform-provider.md b/docs/versioned_docs/version-2.22/workflows/terraform-provider.md new file mode 100644 index 000000000..c7a795d3f --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/terraform-provider.md @@ -0,0 +1,140 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +:::info +On SEV-SNP, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + +::: + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.22/workflows/troubleshooting.md b/docs/versioned_docs/version-2.22/workflows/troubleshooting.md new file mode 100644 index 000000000..903c829e0 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/troubleshooting.md @@ -0,0 +1,200 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` + +### Emergency SSH access + +Emergency SSH access to nodes can be useful to diagnose issues or download important data even if the Kubernetes API isn't reachable anymore. + +1. Enter the `constellation-terraform` directory in your Constellation workspace and enable emergency SSH access to the cluster: + + ```bash + cd constellation-terraform + echo "emergency_ssh = true" >> ./terraform.tfvars + terraform apply + ``` + +2. Sign an existing SSH key with your master secret: + + ```bash + cd ../ # go back to your Constellation workspace + constellation ssh --key your_public_key.pub + ``` + + A certificate is written to `constellation_cert.pub`. + + The certificate is valid for 24 hours and enables you to access your Constellation nodes using + [certificate based authentication](https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Certificate-based_Authentication). + +3. Now you can connect to any Constellation node using your certificate and your private key. + + ```bash + ssh -o CertificateFile=constellation_cert.pub -i root@ + ``` + + Normally, you don't have access to the Constellation nodes since they reside in a private network. + To access those nodes anyways, you can use your Constellation load balancer as a proxy jump host. + For this, use something along the following SSH client configuration: + + ```text + Host + ProxyJump none + + Host * + IdentityFile + PreferredAuthentications publickey + CertificateFile=constellation_cert.pub + User root + ProxyJump + ``` + + With this configuration you can connect to a Constellation node using `ssh -F `. + You can obtain the private node IP and the domain name of the load balancer using your CSP's web UI. diff --git a/docs/versioned_docs/version-2.22/workflows/trusted-launch.md b/docs/versioned_docs/version-2.22/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.22/workflows/upgrade.md b/docs/versioned_docs/version-2.22/workflows/upgrade.md new file mode 100644 index 000000000..3db2ecad6 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting its availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.22/workflows/verify-cli.md b/docs/versioned_docs/version-2.22/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.22/workflows/verify-cluster.md b/docs/versioned_docs/version-2.22/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.22/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.23/_media/SLSA-Badge-full-level3.svg b/docs/versioned_docs/version-2.23/_media/SLSA-Badge-full-level3.svg new file mode 100644 index 000000000..7154d4a13 --- /dev/null +++ b/docs/versioned_docs/version-2.23/_media/SLSA-Badge-full-level3.svg @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_fio_azure_bw.png b/docs/versioned_docs/version-2.23/_media/benchmark_fio_azure_bw.png new file mode 100644 index 000000000..a82ebe2d0 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_fio_azure_bw.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_fio_azure_iops.png b/docs/versioned_docs/version-2.23/_media/benchmark_fio_azure_iops.png new file mode 100644 index 000000000..1723257a8 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_fio_azure_iops.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_fio_gcp_bw.png b/docs/versioned_docs/version-2.23/_media/benchmark_fio_gcp_bw.png new file mode 100644 index 000000000..4f0ecc94b Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_fio_gcp_bw.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_fio_gcp_iops.png b/docs/versioned_docs/version-2.23/_media/benchmark_fio_gcp_iops.png new file mode 100644 index 000000000..571086da2 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_fio_gcp_iops.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_net_p2p_azure.png b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2p_azure.png new file mode 100644 index 000000000..9130349c7 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2p_azure.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_net_p2p_gcp.png b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2p_gcp.png new file mode 100644 index 000000000..a41557e96 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2p_gcp.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_net_p2svc_azure.png b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2svc_azure.png new file mode 100644 index 000000000..d83e17f5a Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2svc_azure.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_net_p2svc_gcp.png b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2svc_gcp.png new file mode 100644 index 000000000..55916a1de Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_net_p2svc_gcp.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/max_latency.png b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/max_latency.png new file mode 100644 index 000000000..696250181 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/max_latency.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/mean_latency.png b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/mean_latency.png new file mode 100644 index 000000000..3b43298ac Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/mean_latency.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/min_latency.png b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/min_latency.png new file mode 100644 index 000000000..1046df67e Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/min_latency.png differ diff --git a/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/p99_latency.png b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/p99_latency.png new file mode 100644 index 000000000..0190118b2 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/benchmark_vault/5replicas/p99_latency.png differ diff --git a/docs/versioned_docs/version-2.23/_media/concept-constellation.svg b/docs/versioned_docs/version-2.23/_media/concept-constellation.svg new file mode 100644 index 000000000..30d32bf6d --- /dev/null +++ b/docs/versioned_docs/version-2.23/_media/concept-constellation.svg @@ -0,0 +1,460 @@ + + diff --git a/docs/versioned_docs/version-2.23/_media/concept-managed.svg b/docs/versioned_docs/version-2.23/_media/concept-managed.svg new file mode 100644 index 000000000..5645a608f --- /dev/null +++ b/docs/versioned_docs/version-2.23/_media/concept-managed.svg @@ -0,0 +1,591 @@ + + diff --git a/docs/versioned_docs/version-2.23/_media/constellation_oneline.svg b/docs/versioned_docs/version-2.23/_media/constellation_oneline.svg new file mode 100644 index 000000000..4e354958a --- /dev/null +++ b/docs/versioned_docs/version-2.23/_media/constellation_oneline.svg @@ -0,0 +1,52 @@ + + + + + + + + diff --git a/docs/versioned_docs/version-2.23/_media/example-emojivoto.jpg b/docs/versioned_docs/version-2.23/_media/example-emojivoto.jpg new file mode 100644 index 000000000..4be0d5b26 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/example-emojivoto.jpg differ diff --git a/docs/versioned_docs/version-2.23/_media/example-online-boutique.jpg b/docs/versioned_docs/version-2.23/_media/example-online-boutique.jpg new file mode 100644 index 000000000..026f0d865 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/example-online-boutique.jpg differ diff --git a/docs/versioned_docs/version-2.23/_media/recovery-gcp-serial-console-link.png b/docs/versioned_docs/version-2.23/_media/recovery-gcp-serial-console-link.png new file mode 100644 index 000000000..eb67f0e99 Binary files /dev/null and b/docs/versioned_docs/version-2.23/_media/recovery-gcp-serial-console-link.png differ diff --git a/docs/versioned_docs/version-2.23/_media/tcb.svg b/docs/versioned_docs/version-2.23/_media/tcb.svg new file mode 100644 index 000000000..e5bcb5b95 --- /dev/null +++ b/docs/versioned_docs/version-2.23/_media/tcb.svg @@ -0,0 +1,535 @@ + + diff --git a/docs/versioned_docs/version-2.23/architecture/attestation.md b/docs/versioned_docs/version-2.23/architecture/attestation.md new file mode 100644 index 000000000..9bd157460 --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/attestation.md @@ -0,0 +1,409 @@ +# Attestation + +This page explains Constellation's attestation process and highlights the cornerstones of its trust model. + +## Terms + +The following lists terms and concepts that help to understand the attestation concept of Constellation. + +### Trusted Platform Module (TPM) + +A TPM chip is a dedicated tamper-resistant crypto-processor. +It can securely store artifacts such as passwords, certificates, encryption keys, or *runtime measurements* (more on this below). +When a TPM is implemented in software, it's typically called a *virtual* TPM (vTPM). + +### Runtime measurement + +A runtime measurement is a cryptographic hash of the memory pages of a so called *runtime component*. Runtime components of interest typically include a system's bootloader or OS kernel. + +### Platform Configuration Register (PCR) + +A Platform Configuration Register (PCR) is a memory location in the TPM that has some unique properties. +To store a new value in a PCR, the existing value is extended with a new value as follows: + +``` +PCR[N] = HASHalg( PCR[N] || ArgumentOfExtend ) +``` + +The PCRs are typically used to store runtime measurements. +The new value of a PCR is always an extension of the existing value. +Thus, storing the measurements of multiple components into the same PCR irreversibly links them together. + +### Measured boot + +Measured boot builds on the concept of chained runtime measurements. +Each component in the boot chain loads and measures the next component into the PCR before executing it. +By comparing the resulting PCR values against trusted reference values, the integrity of the entire boot chain and thereby the running system can be ensured. + +### Remote attestation (RA) + +Remote attestation is the process of verifying certain properties of an application or platform, such as integrity and confidentiality, from a remote location. +In the case of a measured boot, the goal is to obtain a signed attestation statement on the PCR values of the boot measurements. +The statement can then be verified and compared to a set of trusted reference values. +This way, the integrity of the platform can be ensured before sharing secrets with it. + +### Confidential virtual machine (CVM) + +Confidential computing (CC) is the protection of data in-use with hardware-based trusted execution environments (TEEs). +With CVMs, TEEs encapsulate entire virtual machines and isolate them against the hypervisor, other VMs, and direct memory access. +After loading the initial VM image into encrypted memory, the hypervisor calls for a secure processor to measure these initial memory pages. +The secure processor locks these pages and generates an attestation report on the initial page measurements. +CVM memory pages are encrypted with a key that resides inside the secure processor, which makes sure only the guest VM can access them. +The attestation report is signed by the secure processor and can be verified using remote attestation via the certificate authority of the hardware vendor. +Such an attestation statement guarantees the confidentiality and integrity of a CVM. + +### Attested TLS (aTLS) + +In a CC environment, attested TLS (aTLS) can be used to establish secure connections between two parties using the remote attestation features of the CC components. + +aTLS modifies the TLS handshake by embedding an attestation statement into the TLS certificate. +Instead of relying on a certificate authority, aTLS uses this attestation statement to establish trust in the certificate. + +The protocol can be used by clients to verify a server certificate, by a server to verify a client certificate, or for mutual verification (mutual aTLS). + +## Overview + +The challenge for Constellation is to lift a CVM's attestation statement to the Kubernetes software layer and make it end-to-end verifiable. +From there, Constellation needs to expand the attestation from a single CVM to the entire cluster. + +The [*JoinService*](microservices.md#joinservice) and [*VerificationService*](microservices.md#verificationservice) are where all runs together. +Internally, the *JoinService* uses remote attestation to securely join CVM nodes to the cluster. +Externally, the *VerificationService* provides an attestation statement for the cluster's CVMs and configuration. + +The following explains the details of both steps. + +## Node attestation + +The idea is that Constellation nodes should have verifiable integrity from the CVM hardware measurement up to the Kubernetes software layer. +The solution is a verifiable boot chain and an integrity-protected runtime environment. + +Constellation uses measured boot within CVMs, measuring each component in the boot process before executing it. +Outside of CC, this is usually implemented via TPMs. +CVM technologies differ in how they implement runtime measurements, but the general concepts are similar to those of a TPM. +For simplicity, TPM terminology like *PCR* is used in the following. + +When a Constellation node image boots inside a CVM, measured boot is used for all stages and components of the boot chain. +This process goes up to the root filesystem. +The root filesystem is mounted read-only with integrity protection. +For the details on the image and boot stages see the [image architecture](../architecture/images.md) documentation. +Any changes to the image will inevitably also change the corresponding PCR values. +To create a node attestation statement, the Constellation image obtains a CVM attestation statement from the hardware. +This includes the runtime measurements and thereby binds the measured boot results to the CVM hardware measurement. + +In addition to the image measurements, Constellation extends a PCR during the [initialization phase](../workflows/create.md) that irrevocably marks the node as initialized. +The measurement is created using the [*clusterID*](../architecture/keys.md#cluster-identity), tying all future attestation statements to this ID. +Thereby, an attestation statement is unique for every cluster and a node can be identified unambiguously as being initialized. + +To verify an attestation, the hardware's signature and a statement are verified first to establish trust in the contained runtime measurements. +If successful, the measurements are verified against the trusted values of the particular Constellation release version. +Finally, the measurement of the *clusterID* can be compared by calculating it with the [master secret](keys.md#master-secret). + +### Runtime measurements + +Constellation uses runtime measurements to implement the measured boot approach. +As stated above, the underlying hardware technology and guest firmware differ in their implementations of runtime measurements. +The following gives a detailed description of the available measurements in the different cloud environments. + +The runtime measurements consist of two types of values: + +* **Measurements produced by the cloud infrastructure and firmware of the CVM**: +These are measurements of closed-source firmware and other values controlled by the cloud provider. +While not being reproducible for the user, some of them can be compared against previously observed values. +Others may change frequently and aren't suitable for verification. +The [signed image measurements](#chain-of-trust) include measurements that are known, previously observed values. + +* **Measurements produced by the Constellation bootloader and boot chain**: +The Constellation Bootloader takes over from the CVM firmware and [measures the rest of the boot chain](images.md). +The Constellation [Bootstrapper](microservices.md#bootstrapper) is the first user mode component that runs in a Constellation image. +It extends PCR registers with the [IDs](keys.md#cluster-identity) of the cluster marking a node as initialized. + +Constellation allows to specify in the config which measurements should be enforced during the attestation process. +Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. +By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. + + + + +Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | AWS | No | +| 1 | Firmware | AWS | No | +| 2 | Firmware | AWS | No | +| 3 | Firmware | AWS | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | AWS, Constellation Bootloader | Yes | +| 5 | Firmware | AWS | No | +| 6 | Firmware | AWS | No | +| 7 | Secure Boot Policy | AWS, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. +This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [measured boot](https://docs.microsoft.com/en-us/azure/security/fundamentals/measured-boot-host-attestation#measured-boot) verification that's based on the trusted launch feature of [Trusted Launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | Azure | No | +| 1 | Firmware | Azure | No | +| 2 | Firmware | Azure | No | +| 3 | Firmware | Azure | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | Azure, Constellation Bootloader | Yes | +| 5 | Reserved | Azure | No | +| 6 | VM Unique ID | Azure | No | +| 7 | Secure Boot State | Azure, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. +Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +It provides a [launch attestation report](https://cloud.google.com/compute/confidential-vm/docs/monitoring#about_launch_attestation_report_events) that's based on the measured boot feature of [Shielded VMs](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#measured-boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | CVM version and technology | GCP | No | +| 1 | Firmware | GCP | No | +| 2 | Firmware | GCP | No | +| 3 | Firmware | GCP | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | GCP, Constellation Bootloader | Yes | +| 5 | Disk GUID partition table | GCP | No | +| 6 | Disk GUID partition table | GCP | No | +| 7 | GCP Secure Boot Policy | GCP, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +Constellation uses a hypervisor-based vTPM for runtime measurements. + +The vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. +The VMs are attested by obtaining signed PCR values over the VM's boot configuration from the TPM and comparing them to a known, good state (measured boot). + +The following table lists all PCR values of the vTPM and the measured components. +It also lists what components of the boot chain did the measurements and if the value is reproducible and verifiable. +The latter means that the value can be generated offline and compared to the one in the vTPM. + +| PCR | Components | Measured by | Reproducible and verifiable | +| ----------- | ---------------------------------------------------------------- | -------------------------------------- | --------------------------- | +| 0 | Firmware | STACKIT | No | +| 1 | Firmware | STACKIT | No | +| 2 | Firmware | STACKIT | No | +| 3 | Firmware | STACKIT | No | +| 4 | Constellation Bootloader, Kernel, initramfs, Kernel command line | STACKIT, Constellation Bootloader | Yes | +| 5 | Firmware | STACKIT | No | +| 6 | Firmware | STACKIT | No | +| 7 | Secure Boot Policy | STACKIT, Constellation Bootloader | No | +| 8 | - | - | - | +| 9 | initramfs, Kernel command line | Linux Kernel | Yes | +| 10 | User space | Linux IMA | No[^1] | +| 11 | Unified Kernel Image components | Constellation Bootloader | Yes | +| 12 | Reserved | (User space, Constellation Bootloader) | Yes | +| 13 | Reserved | (Constellation Bootloader) | Yes | +| 14 | Secure Boot State | Constellation Bootloader | No | +| 15 | ClusterID | Constellation Bootstrapper | Yes | +| 16–23 | Unused | - | - | + + + + +### CVM verification + +To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. +For verification of the CVM technology, Constellation may expose additional options in its config file. + + + + +On AWS, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* Firmware Signer + + This config option allows you to specify how the firmware signer should be verified. + More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. + You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. + + + + +On GCP, AMD SEV-SNP is used to provide runtime encryption to the VMs. +An SEV-SNP attestation report is used to establish trust in the VM. +You may customize certain parameters for verification of the attestation statement using the Constellation config file. + +* TCB versions + + You can set the minimum version numbers of components in the SEV-SNP TCB. + Use the latest versions to enforce that only machines with the most recent firmware updates are allowed to join the cluster. + Alternatively, you can set a lower minimum version to allow slightly out-of-date machines to still be able to join the cluster. + +* AMD Root Key Certificate + + This certificate is the root of trust for verifying the SEV-SNP certificate chain. + +* AMD Signing Key Certificate + + This is the intermediate certificate for verifying the SEV-SNP report's signature. + If it's not specified, the CLI fetches it from the AMD key distribution server. + + + + +On STACKIT, AMD SEV-ES is used to provide runtime encryption to the VMs. +The hypervisor-based vTPM is used to establish trust in the VM via [runtime measurements](#runtime-measurements). +There is no additional configuration available for STACKIT. + + + + +## Cluster attestation + +Cluster-facing, Constellation's [*JoinService*](microservices.md#joinservice) verifies each node joining the cluster given the configured ground truth runtime measurements. +User-facing, the [*VerificationService*](microservices.md#verificationservice) provides an interface to verify a node using remote attestation. +By verifying the first node during the [initialization](microservices.md#bootstrapper) and configuring the ground truth measurements that are subsequently enforced by the *JoinService*, the whole cluster is verified in a transitive way. + +### Cluster-facing attestation + +The *JoinService* is provided with the runtime measurements of the whitelisted Constellation image version as the ground truth. +During the initialization and the cluster bootstrapping, each node connects to the *JoinService* using [aTLS](#attested-tls-atls). +During the handshake, the node transmits an attestation statement including its runtime measurements. +The *JoinService* verifies that statement and compares the measurements against the ground truth. +For details of the initialization process check the [microservice descriptions](microservices.md). + +After the initialization, every node updates its runtime measurements with the *clusterID* value, marking it irreversibly as initialized. +When an initialized node tries to join another cluster, its measurements inevitably mismatch the measurements of an uninitialized node and it will be declined. + +### User-facing attestation + +The [*VerificationService*](microservices.md#verificationservice) provides an endpoint for obtaining its hardware-based remote attestation statement, which includes the runtime measurements. +A user can [verify](../workflows/verify-cluster.md) this statement and compare the measurements against the configured ground truth and, thus, verify the identity and integrity of all Constellation components and the cluster configuration. Subsequently, the user knows that the entire cluster is in the expected state and is trustworthy. + +## Putting it all together + +This section puts the aforementioned concepts together and illustrate how trust into a Constellation cluster is established and maintained. + +### CLI and node images + +It all starts with the CLI executable. The CLI is signed by Edgeless Systems. To ensure non-repudiability for CLI releases, Edgeless Systems publishes corresponding signatures to the public ledger of the [sigstore project](https://www.sigstore.dev/). There's a [step-by-step guide](../workflows/verify-cli.md) on how to verify CLI signatures based on sigstore. + +The CLI contains the latest runtime measurements of the Constellation node image for all supported cloud platforms. In case a different version of the node image is to be used, the corresponding runtime measurements can be fetched using the CLI's [fetch-measurements command](../reference/cli.md#constellation-config-fetch-measurements). This command downloads the runtime measurements and the corresponding signature from cdn.confidential.cloud. See for example the following files corresponding to node image v2.16.3: + +* [Measurements](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json) +* [Signature](https://cdn.confidential.cloud/constellation/v2/ref/-/stream/stable/v2.16.3/image/measurements.json.sig) + +The CLI contains the long-term public key of Edgeless Systems to verify the signature of downloaded runtime measurements. + +### Cluster creation + +When a cluster is [created](../workflows/create.md), the CLI automatically verifies the runtime measurements of the *first node* using remote attestation. Based on this, the CLI and the first node set up a temporary TLS connection. This [aTLS](#attested-tls-atls) connection is used for two things: + +1. The CLI sends the [master secret](../architecture/keys.md#master-secret) of the to-be-created cluster to the CLI. The master secret is generated by the first node. +2. The first node sends a [kubeconfig file](https://www.redhat.com/sysadmin/kubeconfig) with Kubernetes credentials to the CLI. + +After this, the aTLS connection is closed and the first node bootstraps the Kubernetes cluster. All subsequent interactions between the CLI and the cluster go via the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/) server running inside the cluster. The CLI (and other tools like kubectl) use the credentials referenced by the kubeconfig file to authenticate themselves towards the Kubernetes API server and to establish a mTLS connection. + +The CLI connects to the Kubernetes API to write the runtime measurements for the applicable node image to etcd. The JoinService uses these runtime measurements to verify all nodes that join the cluster subsequently. + +### Chain of trust + +In summary, there's a chain of trust based on cryptographic signatures that goes from the user to the cluster via the CLI. This is illustrated in the following diagram. + +```mermaid +flowchart LR + A[User]-- "verifies" -->B[CLI] + B[CLI]-- "verifies" -->C([Runtime measurements]) + D[Edgeless Systems]-- "signs" -->B[CLI] + D[Edgeless Systems]-- "signs" -->C([Runtime measurements]) + B[CLI]-- "verifies (remote attestation)" -->E[First node] + E[First node]-- "verifies (remote attestation)" -->F[Other nodes] + C([Runtime measurements]) -.-> E[First node] + C([Runtime measurements]) -.-> F[Other nodes] +``` + +### Upgrades + +Whenever a cluster is [upgraded](../workflows/upgrade.md) to a new version of the node image, the CLI sends the corresponding runtime measurements via the Kubernetes API server. The new runtime measurements are stored in etcd within the cluster and replace any previous runtime measurements. The new runtime measurements are then used automatically by the JoinService for the verification of new nodes. + +## References + +[^1]: Linux IMA produces runtime measurements of user-space binaries. +However, these measurements aren't deterministic and thus, PCR\[10] can't be compared to a constant value. +Instead, a policy engine must be used to verify the TPM event log against a policy. diff --git a/docs/versioned_docs/version-2.23/architecture/encrypted-storage.md b/docs/versioned_docs/version-2.23/architecture/encrypted-storage.md new file mode 100644 index 000000000..f047fa4a9 --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/encrypted-storage.md @@ -0,0 +1,62 @@ +# Encrypted persistent storage + +Confidential VMs provide runtime memory encryption to protect data in use. +In the context of Kubernetes, this is sufficient for the confidentiality and integrity of stateless services. +Consider a front-end web server, for example, that keeps all connection information cached in main memory. +No sensitive data is ever written to an insecure medium. +However, many real-world applications need some form of state or data-lake service that's connected to a persistent storage device and requires encryption at rest. +As described in [Use persistent storage](../workflows/storage.md), cloud service providers (CSPs) use the container storage interface (CSI) to make their storage solutions available to Kubernetes workloads. +These CSI storage solutions often support some sort of encryption. +For example, Google Cloud [encrypts data at rest by default](https://cloud.google.com/security/encryption/default-encryption), without any action required by the customer. + +## Cloud provider-managed encryption + +CSP-managed storage solutions encrypt the data in the cloud backend before writing it physically to disk. +In the context of confidential computing and Constellation, the CSP and its managed services aren't trusted. +Hence, cloud provider-managed encryption protects your data from offline hardware access to physical storage devices. +It doesn't protect it from anyone with infrastructure-level access to the storage backend or a malicious insider in the cloud platform. +Even with "bring your own key" or similar concepts, the CSP performs the encryption process with access to the keys and plaintext data. + +In the security model of Constellation, securing persistent storage and thereby data at rest requires that all cryptographic operations are performed inside a trusted execution environment. +Consequently, using CSP-managed encryption of persistent storage usually isn't an option. + +## Constellation-managed encryption + +Constellation provides CSI drivers for storage solutions in all major clouds with built-in encryption support. +Block storage provisioned by the CSP is [mapped](https://guix.gnu.org/manual/en/html_node/Mapped-Devices.html) using the [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html), and optionally the [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html), kernel modules, before it's formatted and accessed by the Kubernetes workloads. +All cryptographic operations happen inside the trusted environment of the confidential Constellation node. + +Note that for integrity-protected disks, [volume expansion](https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/) isn't supported. + +By default the driver uses data encryption keys (DEKs) issued by the Constellation [*KeyService*](microservices.md#keyservice). +The DEKs are in turn derived from the Constellation's key encryption key (KEK), which is directly derived from the [master secret](keys.md#master-secret). +This is the recommended mode of operation, and also requires the least amount of setup by the cluster administrator. + +Alternatively, the driver can be configured to use a key management system to store and access KEKs and DEKs. + +Refer to [keys and cryptography](keys.md) for more details on key management in Constellation. + +Once deployed and configured, the CSI driver ensures transparent encryption and integrity of all persistent volumes provisioned via its storage class. +Data at rest is secured without any additional actions required by the developer. + +## Cryptographic algorithms + +This section gives an overview of the libraries, cryptographic algorithms, and their configurations, used in Constellation's CSI drivers. + +### dm-crypt + +To interact with the dm-crypt kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +New devices are formatted as [LUKS2](https://gitlab.com/cryptsetup/LUKS2-docs/-/tree/master) partitions with a sector size of 4096 bytes. +The used key derivation function is [Argon2id](https://datatracker.ietf.org/doc/html/rfc9106) with the [recommended parameters for memory-constrained environments](https://datatracker.ietf.org/doc/html/rfc9106#section-7.4) of 3 iterations and 64 MiB of memory, utilizing 4 parallel threads. +For encryption Constellation uses AES in XTS-Plain64. The key size is 512 bit. + +### dm-integrity + +To interact with the dm-integrity kernel module, Constellation uses [libcryptsetup](https://gitlab.com/cryptsetup/cryptsetup/). +When enabled, the used data integrity algorithm is [HMAC](https://datatracker.ietf.org/doc/html/rfc2104) with SHA256 as the hash function. +The tag size is 32 Bytes. + +## Encrypted S3 object storage + +Constellation comes with a service that you can use to transparently retrofit client-side encryption to existing applications that use S3 (AWS or compatible) for storage. +To learn more, check out the [s3proxy documentation](../workflows/s3proxy.md). diff --git a/docs/versioned_docs/version-2.23/architecture/images.md b/docs/versioned_docs/version-2.23/architecture/images.md new file mode 100644 index 000000000..8a9c51d36 --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/images.md @@ -0,0 +1,49 @@ +# Constellation images + +Constellation uses a minimal version of Fedora as the operating system running inside confidential VMs. This Linux distribution is optimized for containers and designed to be stateless. +The Constellation images provide measured boot and an immutable filesystem. + +## Measured boot + +```mermaid +flowchart LR + Firmware --> Bootloader + Bootloader --> uki + subgraph uki[Unified Kernel Image] + Kernel[Kernel] + initramfs[Initramfs] + cmdline[Kernel Command Line] + end + uki --> rootfs[Root Filesystem] +``` + +Measured boot uses a Trusted Platform Module (TPM) to measure every part of the boot process. This allows for verification of the integrity of a running system at any point in time. To ensure correct measurements of every stage, each stage is responsible to measure the next stage before transitioning. + +### Firmware + +With confidential VMs, the firmware is the root of trust and is measured automatically at boot. After initialization, the firmware will load and measure the bootloader before executing it. + +### Bootloader + +The bootloader is the first modifiable part of the boot chain. The bootloader is tasked with loading the kernel, initramfs and setting the kernel command line. The Constellation bootloader measures these components before starting the kernel. + +### initramfs + +The initramfs is a small filesystem loaded to prepare the actual root filesystem. The Constellation initramfs maps the block device containing the root filesystem with [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html). The initramfs then mounts the root filesystem from the mapped block device. + +dm-verity provides integrity checking using a cryptographic hash tree. When a block is read, its integrity is checked by verifying the tree against a trusted root hash. The initramfs reads this root hash from the previously measured kernel command line. Thus, if any block of the root filesystem's device is modified on disk, trying to read the modified block will result in a kernel panic at runtime. + +After mounting the root filesystem, the initramfs will switch over and start the `init` process of the integrity-protected root filesystem. + +## State disk + +In addition to the read-only root filesystem, each Constellation node has a disk for storing state data. +This disk is mounted readable and writable by the initramfs and contains data that should persist across reboots. +Such data can contain sensitive information and, therefore, must be stored securely. +To that end, the state disk is protected by authenticated encryption. +See the section on [keys and encryption](keys.md#storage-encryption) for more information on the cryptographic primitives in use. + +## Kubernetes components + +During initialization, the [*Bootstrapper*](microservices.md#bootstrapper) downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) as configured by the user. +They're stored on the state partition and can be updated once new releases need to be installed. diff --git a/docs/versioned_docs/version-2.23/architecture/keys.md b/docs/versioned_docs/version-2.23/architecture/keys.md new file mode 100644 index 000000000..49821cd0b --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/keys.md @@ -0,0 +1,130 @@ +# Key management and cryptographic primitives + +Constellation protects and isolates your cluster and workloads. +To that end, cryptography is the foundation that ensures the confidentiality and integrity of all components. +Evaluating the security and compliance of Constellation requires a precise understanding of the cryptographic primitives and keys used. +The following gives an overview of the architecture and explains the technical details. + +## Confidential VMs + +Confidential VM (CVM) technology comes with hardware and software components for memory encryption, isolation, and remote attestation. +For details on the implementations and cryptographic soundness, refer to the hardware vendors' documentation and advisories. + +## Master secret + +The master secret is the cryptographic material used for deriving the [*clusterID*](#cluster-identity) and the *key encryption key (KEK)* for [storage encryption](#storage-encryption). +It's generated during the bootstrapping of a Constellation cluster. +It can either be managed by [Constellation](#constellation-managed-key-management) or an [external key management system](#user-managed-key-management). +In case of [recovery](#recovery-and-migration), the master secret allows to decrypt the state and recover a Constellation cluster. + +## Cluster identity + +The identity of a Constellation cluster is represented by cryptographic [measurements](attestation.md#runtime-measurements): + +The **base measurements** represent the identity of a valid, uninitialized Constellation node. +They depend on the node image, but are otherwise the same for every Constellation cluster. +On node boot, they're determined using the CVM's attestation mechanism and [measured boot up to the read-only root filesystem](images.md). + +The **clusterID** represents the identity of a single initialized Constellation cluster. +It's derived from the master secret and a cryptographically random salt and unique for every Constellation cluster. +The [Bootstrapper](microservices.md#bootstrapper) measures the *clusterID* into its own PCR before executing any code not measured as part of the *base measurements*. +See [Node attestation](attestation.md#node-attestation) for details. + +The remote attestation statement of a Constellation cluster combines the *base measurements* and the *clusterID* for a verifiable, unspoofable, unique identity. + +## Network encryption + +Constellation encrypts all cluster network communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +See [network encryption](networking.md) for more details. + +The Cilium agent running on each node establishes a secure [WireGuard](https://www.wireguard.com/) tunnel between it and all other known nodes in the cluster. +Each node creates its own [Curve25519](http://cr.yp.to/ecdh.html) encryption key pair and distributes its public key via Kubernetes. +A node uses another node's public key to decrypt and encrypt traffic from and to Cilium-managed endpoints running on that node. +Connections are always encrypted peer-to-peer using [ChaCha20](http://cr.yp.to/chacha.html) with [Poly1305](http://cr.yp.to/mac.html). +WireGuard implements [forward secrecy with key rotation every 2 minutes](https://lists.zx2c4.com/pipermail/wireguard/2017-December/002141.html). + +## Storage encryption + +Constellation supports transparent encryption of persistent storage. +The Linux kernel's device mapper-based encryption features are used to encrypt the data on the block storage level. +Currently, the following primitives are used for block storage encryption: + +* [dm-crypt](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-crypt.html) +* [dm-integrity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/dm-integrity.html) + +Adding primitives for integrity protection in the CVM attacker model are under active development and will be available in a future version of Constellation. +See [encrypted storage](encrypted-storage.md) for more details. + +As a cluster administrator, when creating a cluster, you can use the Constellation [installation program](orchestration.md) to select one of the following methods for key management: + +* Constellation-managed key management +* User-managed key management + +### Constellation-managed key management + +#### Key material and key derivation + +During the creation of a Constellation cluster, the cluster's master secret is used to derive a KEK. +This means creating two clusters with the same master secret will yield the same KEK. +Any data encryption key (DEK) is derived from the KEK via HKDF. +Note that the master secret is recommended to be unique for every cluster and shouldn't be reused (except in case of [recovering](../workflows/recovery.md) a cluster). + +#### State and storage + +The KEK is derived from the master secret during the initialization. +Subsequently, all other key material is derived from the KEK. +Given the same KEK, any DEK can be derived deterministically from a given identifier. +Hence, there is no need to store DEKs. They can be derived on demand. +After the KEK was derived, it's stored in memory only and never leaves the CVM context. + +#### Availability + +Constellation-managed key management has the same availability as the underlying Kubernetes cluster. +Therefore, the KEK is stored in the [distributed Kubernetes etcd storage](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) to allow for unexpected but non-fatal (control-plane) node failure. +The etcd storage is backed by the encrypted and integrity protected [state disk](images.md#state-disk) of the nodes. + +#### Recovery + +Constellation clusters can be recovered in the event of a disaster, even when all node machines have been stopped and need to be rebooted. +For details on the process see the [recovery workflow](../workflows/recovery.md). + +### User-managed key management + +User-managed key management is under active development and will be available soon. +In scenarios where constellation-managed key management isn't an option, this mode allows you to keep full control of your keys. +For example, compliance requirements may force you to keep your KEKs in an on-prem key management system (KMS). + +During the creation of a Constellation cluster, you specify a KEK present in a remote KMS. +This follows the common scheme of "bring your own key" (BYOK). +Constellation will support several KMSs for managing the storage and access of your KEK. +Initially, it will support the following KMSs: + +* [AWS KMS](https://aws.amazon.com/kms/) +* [GCP KMS](https://cloud.google.com/security-key-management) +* [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) +* [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) + +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). +In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. +Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. + +KMIP support allows you to use your KMIP-compatible on-prem KMS and keep full control over your keys. +This follows the common scheme of "hold your own key" (HYOK). + +The KEK is used to encrypt per-data "data encryption keys" (DEKs). +DEKs are generated to encrypt your data before storing it on persistent storage. +After being encrypted by the KEK, the DEKs are stored on dedicated cloud storage for persistence. +Currently, Constellation supports the following cloud storage options: + +* [AWS S3](https://aws.amazon.com/s3/) +* [GCP Cloud Storage](https://cloud.google.com/storage) +* [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/#overview) + +The DEKs are only present in plaintext form in the encrypted main memory of the CVMs. +Similarly, the cryptographic operations for encrypting data before writing it to persistent storage are performed in the context of the CVMs. + +#### Recovery and migration + +In the case of a disaster, the KEK can be used to decrypt the DEKs locally and subsequently use them to decrypt and retrieve the data. +In case of migration, configuring the same KEK will provide seamless migration of data. +Thus, only the DEK storage needs to be transferred to the new cluster alongside the encrypted data for seamless migration. diff --git a/docs/versioned_docs/version-2.23/architecture/microservices.md b/docs/versioned_docs/version-2.23/architecture/microservices.md new file mode 100644 index 000000000..90bae783b --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/microservices.md @@ -0,0 +1,73 @@ +# Microservices + +Constellation takes care of bootstrapping and initializing a Confidential Kubernetes cluster. +During the lifetime of the cluster, it handles day 2 operations such as key management, remote attestation, and updates. +These features are provided by several microservices: + +* The [Bootstrapper](microservices.md#bootstrapper) initializes a Constellation node and bootstraps the cluster +* The [JoinService](microservices.md#joinservice) joins new nodes to an existing cluster +* The [VerificationService](microservices.md#verificationservice) provides remote attestation functionality +* The [KeyService](microservices.md#keyservice) manages Constellation-internal keys + +The relations between microservices are shown in the following diagram: + +```mermaid +flowchart LR + subgraph admin [Admin's machine] + A[Constellation CLI] + end + subgraph img [Constellation OS image] + B[Constellation OS] + C[Bootstrapper] + end + subgraph Kubernetes + D[JoinService] + E[KeyService] + F[VerificationService] + end + A -- deploys --> + B -- starts --> C + C -- deploys --> D + C -- deploys --> E + C -- deploys --> F +``` + +## Bootstrapper + +The *Bootstrapper* is the first microservice launched after booting a Constellation node image. +It sets up that machine as a Kubernetes node and integrates that node into the Kubernetes cluster. +To this end, the *Bootstrapper* first downloads and verifies the [Kubernetes components](https://kubernetes.io/docs/concepts/overview/components/) at the configured versions. +The *Bootstrapper* tries to find an existing cluster and if successful, communicates with the [JoinService](microservices.md#joinservice) to join the node. +Otherwise, it waits for an initialization request to create a new Kubernetes cluster. + +## JoinService + +The *JoinService* runs as [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) on each control-plane node. +New nodes (at cluster start, or later through autoscaling) send a request to the service over [attested TLS (aTLS)](attestation.md#attested-tls-atls). +The *JoinService* verifies the new node's certificate and attestation statement. +If attestation is successful, the new node is supplied with an encryption key from the [*KeyService*](microservices.md#keyservice) for its state disk, and a Kubernetes bootstrap token. + + +```mermaid +sequenceDiagram + participant New node + participant JoinService + New node->>JoinService: aTLS handshake (server side verification) + JoinService-->>New node: # + New node->>+JoinService: IssueJoinTicket(DiskUUID, NodeName, IsControlPlane) + JoinService->>+KeyService: GetDataKey(DiskUUID) + KeyService-->>-JoinService: DiskEncryptionKey + JoinService-->>-New node: DiskEncryptionKey, KubernetesJoinToken, ... +``` + +## VerificationService + +The *VerificationService* runs as DaemonSet on each node. +It provides user-facing functionality for remote attestation during the cluster's lifetime via an endpoint for [verifying the cluster](attestation.md#cluster-attestation). +Read more about the hardware-based [attestation feature](attestation.md) of Constellation and how to [verify](../workflows/verify-cluster.md) a cluster on the client side. + +## KeyService + +The *KeyService* runs as DaemonSet on each control-plane node. +It implements the key management for the [storage encryption keys](keys.md#storage-encryption) in Constellation. These keys are used for the [state disk](images.md#state-disk) of each node and the [transparently encrypted storage](encrypted-storage.md) for Kubernetes. +Depending on wether the [constellation-managed](keys.md#constellation-managed-key-management) or [user-managed](keys.md#user-managed-key-management) mode is used, the *KeyService* holds the key encryption key (KEK) directly or calls an external key management service (KMS) for key derivation respectively. diff --git a/docs/versioned_docs/version-2.23/architecture/networking.md b/docs/versioned_docs/version-2.23/architecture/networking.md new file mode 100644 index 000000000..e9cbdf029 --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/networking.md @@ -0,0 +1,22 @@ +# Network encryption + +Constellation encrypts all pod communication using the [container network interface (CNI)](https://github.com/containernetworking/cni). +To that end, Constellation deploys, configures, and operates the [Cilium](https://cilium.io/) CNI plugin. +Cilium provides [transparent encryption](https://docs.cilium.io/en/stable/security/network/encryption) for all cluster traffic using either IPSec or [WireGuard](https://www.wireguard.com/). +Currently, Constellation only supports WireGuard as the encryption engine. +You can read more about the cryptographic soundness of WireGuard [in their white paper](https://www.wireguard.com/papers/wireguard.pdf). + +Cilium is actively working on implementing a feature called [`host-to-host`](https://github.com/cilium/cilium/pull/19401) encryption mode for WireGuard. +With `host-to-host`, all traffic between nodes will be tunneled via WireGuard (host-to-host, host-to-pod, pod-to-host, pod-to-pod). +Until the `host-to-host` feature is released, Constellation enables `pod-to-pod` encryption. +This mode encrypts all traffic between Kubernetes pods using WireGuard tunnels. + +When using Cilium in the default setup but with encryption enabled, there is a [known issue](https://docs.cilium.io/en/v1.12/gettingstarted/encryption/#egress-traffic-to-not-yet-discovered-remote-endpoints-may-be-unencrypted) +that can cause pod-to-pod traffic to be unencrypted. +To mitigate this issue, Constellation adds a *strict* mode to Cilium's `pod-to-pod` encryption. +This mode changes the default behavior of traffic that's destined for an unknown endpoint to not be send out in plaintext, but instead being dropped. +The strict mode distinguishes between traffic that's send to a pod from traffic that's destined for a cluster-external endpoint by considering the pod's CIDR range. + +Traffic originating from hosts isn't encrypted yet. +This mainly includes health checks from Kubernetes API server. +Also, traffic proxied over the API server via e.g. `kubectl port-forward` isn't encrypted. diff --git a/docs/versioned_docs/version-2.23/architecture/observability.md b/docs/versioned_docs/version-2.23/architecture/observability.md new file mode 100644 index 000000000..0f4daffd4 --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/observability.md @@ -0,0 +1,74 @@ +# Observability + +In Kubernetes, observability is the ability to gain insight into the behavior and performance of applications. +It helps identify and resolve issues more effectively, ensuring stability and performance of Kubernetes workloads, reducing downtime and outages, and improving efficiency. +The "three pillars of observability" are logs, metrics, and traces. + +In the context of Confidential Computing, observability is a delicate subject and needs to be applied such that it doesn't leak any sensitive information. +The following gives an overview of where and how you can apply standard observability tools in Constellation. + +## Cloud resource monitoring + +While inaccessible, Constellation's nodes are still visible as black box VMs to the hypervisor. +Resource consumption, such as memory and CPU utilization, can be monitored from the outside and observed via the cloud platforms directly. +Similarly, other resources, such as storage and network and their respective metrics, are visible via the cloud platform. + +## Metrics + +Metrics are numeric representations of data measured over intervals of time. They're essential for understanding system health and gaining insights using telemetry signals. + +By default, Constellation exposes the [metrics for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/) inside the cluster. +Similarly, the [etcd metrics](https://etcd.io/docs/v3.5/metrics/) endpoints are exposed inside the cluster. +These [metrics endpoints can be disabled](https://kubernetes.io/docs/concepts/cluster-administration/system-metrics/#disabling-metrics). + +You can collect these cluster-internal metrics via tools such as [Prometheus](https://prometheus.io/) or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +Constellation's CNI Cilium also supports [metrics via Prometheus endpoints](https://docs.cilium.io/en/latest/observability/metrics/). +However, in Constellation, they're disabled by default and must be enabled first. + +## Logs + +Logs represent discrete events that usually describe what's happening with your service. +The payload is an actual message emitted from your system along with a metadata section containing a timestamp, labels, and tracking identifiers. + +### System logs + +Detailed system-level logs are accessible via `/var/log` and [journald](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html) on the nodes directly. +They can be collected from there, for example, via [Filebeat and Logstash](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html), which are tools of the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +In case of an error during the initialization, the CLI automatically collects the [Bootstrapper](./microservices.md#bootstrapper) logs and returns these as a file for [troubleshooting](../workflows/troubleshooting.md). Here is an example of such an event: + +```shell-session +Cluster initialization failed. This error is not recoverable. +Terminate your cluster and try again. +Fetched bootstrapper logs are stored in "constellation-cluster.log" +``` + +### Kubernetes logs + +Constellation supports the [Kubernetes logging architecture](https://kubernetes.io/docs/concepts/cluster-administration/logging/). +By default, logs are written to the nodes' encrypted state disks. +These include the Pod and container logs and the [system component logs](https://kubernetes.io/docs/concepts/cluster-administration/logging/#system-component-logs). + +[Constellation services](microservices.md) run as Pods inside the `kube-system` namespace and use the standard container logging mechanism. +The same applies for the [Cilium Pods](https://docs.cilium.io/en/latest/operations/troubleshooting/#logs). + +You can collect logs from within the cluster via tools such as [Fluentd](https://github.com/fluent/fluentd), [Loki](https://github.com/grafana/loki), or the [Elastic Stack](https://www.elastic.co/de/elastic-stack/). + +## Traces + +Modern systems are implemented as interconnected complex and distributed microservices. Understanding request flows and system communications is challenging, mainly because all systems in a chain need to be modified to propagate tracing information. Distributed tracing is a new approach to increasing observability and understanding performance bottlenecks. A trace represents consecutive events that reflect an end-to-end request path in a distributed system. + +Constellation supports [traces for Kubernetes system components](https://kubernetes.io/docs/concepts/cluster-administration/system-traces/). +By default, they're disabled and need to be enabled first. + +Similarly, Cilium can be enabled to [export traces](https://cilium.io/use-cases/metrics-export/). + +You can collect these traces via tools such as [Jaeger](https://www.jaegertracing.io/) or [Zipkin](https://zipkin.io/). + +## Integrations + +Platforms and SaaS solutions such as Datadog, logz.io, Dynatrace, or New Relic facilitate the observability challenge for Kubernetes and provide all-in-one SaaS solutions. +They install agents into the cluster that collect metrics, logs, and tracing information and upload them into the data lake of the platform. +Technically, the agent-based approach is compatible with Constellation, and attaching these platforms is straightforward. +However, you need to evaluate if the exported data might violate Constellation's compliance and privacy guarantees by uploading them to a third-party platform. diff --git a/docs/versioned_docs/version-2.23/architecture/orchestration.md b/docs/versioned_docs/version-2.23/architecture/orchestration.md new file mode 100644 index 000000000..3c8d529e7 --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/orchestration.md @@ -0,0 +1,83 @@ +# Orchestrating Constellation clusters + +You can use the CLI to create a cluster on the supported cloud platforms. +The CLI provisions the resources in your cloud environment and initiates the initialization of your cluster. +It uses a set of parameters and an optional configuration file to manage your cluster installation. +The CLI is also used for updating your cluster. + +## Workspaces + +Each Constellation cluster has an associated *workspace*. +The workspace is where data such as the Constellation state and config files are stored. +Each workspace is associated with a single cluster and configuration. +The CLI stores state in the local filesystem making the current directory the active workspace. +Multiple clusters require multiple workspaces, hence, multiple directories. +Note that every operation on a cluster always has to be performed from the directory associated with its workspace. + +You may copy files from the workspace to other locations, +but you shouldn't move or delete them while the cluster is still being used. +The Constellation CLI takes care of managing the workspace. +Only when a cluster was terminated, and you are sure the files aren't needed anymore, should you remove a workspace. + +## Cluster creation process + +To allow for fine-grained configuration of your cluster and cloud environment, Constellation supports an extensive configuration file with strong defaults. [Generating the configuration file](../workflows/config.md) is typically the first thing you do in the workspace. + +Altogether, the following files are generated during the creation of a Constellation cluster and stored in the current workspace: + +* a configuration file +* a state file +* a Base64-encoded master secret +* [Terraform artifacts](../reference/terraform.md), stored in subdirectories +* a Kubernetes `kubeconfig` file. + +After the initialization of your cluster, the CLI will provide you with a Kubernetes `kubeconfig` file. +This file grants you access to your Kubernetes cluster and configures the [kubectl](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) tool. +In addition, the cluster's [identifier](orchestration.md#post-installation-configuration) is returned and stored in the state file. + +### Creation process details + +1. The CLI `apply` command first creates the confidential VM (CVM) resources in your cloud environment and configures the network +2. Each CVM boots the Constellation node image and measures every component in the boot chain +3. The first microservice launched in each node is the [*Bootstrapper*](microservices.md#bootstrapper) +4. The *Bootstrapper* waits until it either receives an initialization request or discovers an initialized cluster +5. The CLI then connects to the *Bootstrapper* of a selected node, sends the configuration, and initiates the initialization of the cluster +6. The *Bootstrapper* of **that** node [initializes the Kubernetes cluster](microservices.md#bootstrapper) and deploys the other Constellation [microservices](microservices.md) including the [*JoinService*](microservices.md#joinservice) +7. Subsequently, the *Bootstrappers* of the other nodes discover the initialized cluster and send join requests to the *JoinService* +8. As part of the join request each node includes an attestation statement of its boot measurements as authentication +9. The *JoinService* verifies the attestation statements and joins the nodes to the Kubernetes cluster +10. This process is repeated for every node joining the cluster later (e.g., through autoscaling) + +## Post-installation configuration + +Post-installation the CLI provides a configuration for [accessing the cluster using the Kubernetes API](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/). +The `kubeconfig` file provides the credentials and configuration for connecting and authenticating to the API server. +Once configured, orchestrate the Kubernetes cluster via `kubectl`. + +After the initialization, the CLI will present you with a couple of tokens: + +* The [*master secret*](keys.md#master-secret) (stored in the `constellation-mastersecret.json` file by default) +* The [*clusterID*](keys.md#cluster-identity) of your cluster in Base64 encoding + +You can read more about these values and their meaning in the guide on [cluster identity](keys.md#cluster-identity). + +The *master secret* must be kept secret and can be used to [recover your cluster](../workflows/recovery.md). +Instead of managing this secret manually, you can [use your key management solution of choice](keys.md#user-managed-key-management) with Constellation. + +The *clusterID* uniquely identifies a cluster and can be used to [verify your cluster](../workflows/verify-cluster.md). + +## Upgrades + +Constellation images and microservices may need to be upgraded to new versions during the lifetime of a cluster. +Constellation implements a rolling update mechanism ensuring no downtime of the control or data plane. +You can upgrade a Constellation cluster with a single operation by using the CLI. +For step-by-step instructions on how to do this, refer to [Upgrade your cluster](../workflows/upgrade.md). + +### Attestation of upgrades + +With every new image, corresponding measurements are released. +During an update procedure, the CLI provides new measurements to the [JoinService](microservices.md#joinservice) securely. +New measurements for an updated image are automatically pulled and verified by the CLI following the [supply chain security concept](attestation.md#chain-of-trust) of Constellation. +The [attestation section](attestation.md#cluster-facing-attestation) describes in detail how these measurements are then used by the JoinService for the attestation of nodes. + + diff --git a/docs/versioned_docs/version-2.23/architecture/overview.md b/docs/versioned_docs/version-2.23/architecture/overview.md new file mode 100644 index 000000000..386f93b2f --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/overview.md @@ -0,0 +1,30 @@ +# Overview + +Constellation is a cloud-based confidential orchestration platform. +The foundation of Constellation is Kubernetes and therefore shares the same technology stack and architecture principles. +To learn more about Constellation and Kubernetes, see [product overview](../overview/product.md). + +## About orchestration and updates + +As a cluster administrator, you can use the [Constellation CLI](orchestration.md) to install and deploy a cluster. +Updates are provided in accordance with the [support policy](versions.md). + +## About microservices and attestation + +Constellation manages the nodes and network in your cluster. All nodes are bootstrapped by the [*Bootstrapper*](microservices.md#bootstrapper). They're verified and authenticated by the [*JoinService*](microservices.md#joinservice) before being added to the cluster and the network. Finally, the entire cluster can be verified via the [*VerificationService*](microservices.md#verificationservice) using [remote attestation](attestation.md). + +## About node images and verified boot + +Constellation comes with operating system images for Kubernetes control-plane and worker nodes. +They're highly optimized for running containerized workloads and specifically prepared for running inside confidential VMs. +You can learn more about [the images](images.md) and how verified boot ensures their integrity during boot and beyond. + +## About key management and cryptographic primitives + +Encryption of data at-rest, in-transit, and in-use is the fundamental building block for confidential computing and Constellation. Learn more about the [keys and cryptographic primitives](keys.md) used in Constellation, [encrypted persistent storage](encrypted-storage.md), and [network encryption](networking.md). + +## About observability + +Observability in Kubernetes refers to the capability to troubleshoot issues using telemetry signals such as logs, metrics, and traces. +In the realm of Confidential Computing, it's crucial that observability aligns with confidentiality, necessitating careful implementation. +Learn more about the [observability capabilities in Constellation](./observability.md). diff --git a/docs/versioned_docs/version-2.23/architecture/versions.md b/docs/versioned_docs/version-2.23/architecture/versions.md new file mode 100644 index 000000000..d5cbc987b --- /dev/null +++ b/docs/versioned_docs/version-2.23/architecture/versions.md @@ -0,0 +1,21 @@ +# Versions and support policy + +All components of Constellation use a three-digit version number of the form `v..`. +The components are released in lock step, usually on the first Tuesday of every month. This release primarily introduces new features, but may also include security or performance improvements. The `MINOR` version will be incremented as part of this release. + +Additional `PATCH` releases may be created on demand, to fix security issues or bugs before the next `MINOR` release window. + +New releases are published on [GitHub](https://github.com/edgelesssys/constellation/releases). + +## Kubernetes support policy + +Constellation is aligned to the [version support policy of Kubernetes](https://kubernetes.io/releases/version-skew-policy/#supported-versions), and therefore usually supports the most recent three minor versions. +When a new minor version of Kubernetes is released, support is added to the next Constellation release, and that version then supports four Kubernetes versions. +Subsequent Constellation releases drop support for the oldest (and deprecated) Kubernetes version. + +The following Kubernetes versions are currently supported: + + +* v1.29.15 +* v1.30.12 +* v1.31.8 diff --git a/docs/versioned_docs/version-2.23/getting-started/examples.md b/docs/versioned_docs/version-2.23/getting-started/examples.md new file mode 100644 index 000000000..fded84980 --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/examples.md @@ -0,0 +1,6 @@ +# Examples + +After you [installed the CLI](install.md) and [created your first cluster](first-steps.md), you're ready to deploy applications. Why not start with one of the following examples? +* [Emojivoto](examples/emojivoto.md): a simple but fun web application +* [Online Boutique](examples/online-boutique.md): an e-commerce demo application by Google consisting of 11 separate microservices +* [Horizontal Pod Autoscaling](examples/horizontal-scaling.md): an example demonstrating Constellation's autoscaling capabilities diff --git a/docs/versioned_docs/version-2.23/getting-started/examples/emojivoto.md b/docs/versioned_docs/version-2.23/getting-started/examples/emojivoto.md new file mode 100644 index 000000000..2bbe27917 --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/examples/emojivoto.md @@ -0,0 +1,22 @@ +# Emojivoto +[Emojivoto](https://github.com/BuoyantIO/emojivoto) is a simple and fun application that's well suited to test the basic functionality of your cluster. + + + +emojivoto - Web UI + + + +1. Deploy the application: + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` +2. Wait until it becomes available: + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + ``` +3. Forward the web service to your machine: + ```bash + kubectl -n emojivoto port-forward svc/web-svc 8080:80 + ``` +4. Visit [http://localhost:8080](http://localhost:8080) diff --git a/docs/versioned_docs/version-2.23/getting-started/examples/filestash-s3proxy.md b/docs/versioned_docs/version-2.23/getting-started/examples/filestash-s3proxy.md new file mode 100644 index 000000000..b9a394256 --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/examples/filestash-s3proxy.md @@ -0,0 +1,107 @@ + +# Deploying Filestash + +Filestash is a web frontend for different storage backends, including S3. +It's a useful application to showcase s3proxy in action. + +1. Deploy s3proxy as described in [Deployment](../../workflows/s3proxy.md#deployment). +2. Create a deployment file for Filestash with one pod: + +```sh +cat << EOF > "deployment-filestash.yaml" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filestash +spec: + replicas: 1 + selector: + matchLabels: + app: filestash + template: + metadata: + labels: + app: filestash + spec: + hostAliases: + - ip: $(kubectl get svc s3proxy-service -o=jsonpath='{.spec.clusterIP}') + hostnames: + - "s3.us-east-1.amazonaws.com" + - "s3.us-east-2.amazonaws.com" + - "s3.us-west-1.amazonaws.com" + - "s3.us-west-2.amazonaws.com" + - "s3.eu-north-1.amazonaws.com" + - "s3.eu-south-1.amazonaws.com" + - "s3.eu-south-2.amazonaws.com" + - "s3.eu-west-1.amazonaws.com" + - "s3.eu-west-2.amazonaws.com" + - "s3.eu-west-3.amazonaws.com" + - "s3.eu-central-1.amazonaws.com" + - "s3.eu-central-2.amazonaws.com" + - "s3.ap-northeast-1.amazonaws.com" + - "s3.ap-northeast-2.amazonaws.com" + - "s3.ap-northeast-3.amazonaws.com" + - "s3.ap-east-1.amazonaws.com" + - "s3.ap-southeast-1.amazonaws.com" + - "s3.ap-southeast-2.amazonaws.com" + - "s3.ap-southeast-3.amazonaws.com" + - "s3.ap-southeast-4.amazonaws.com" + - "s3.ap-south-1.amazonaws.com" + - "s3.ap-south-2.amazonaws.com" + - "s3.me-south-1.amazonaws.com" + - "s3.me-central-1.amazonaws.com" + - "s3.il-central-1.amazonaws.com" + - "s3.af-south-1.amazonaws.com" + - "s3.ca-central-1.amazonaws.com" + - "s3.sa-east-1.amazonaws.com" + containers: + - name: filestash + image: machines/filestash:latest + ports: + - containerPort: 8334 + volumeMounts: + - name: ca-cert + mountPath: /etc/ssl/certs/kube-ca.crt + subPath: kube-ca.crt + volumes: + - name: ca-cert + secret: + secretName: s3proxy-tls + items: + - key: ca.crt + path: kube-ca.crt +EOF +``` + +The pod spec includes the `hostAliases` key, which adds an entry to the pod's `/etc/hosts`. +The entry forwards all requests for any of the currently defined AWS regions to the Kubernetes service `s3proxy-service`. +If you followed the s3proxy [Deployment](../../workflows/s3proxy.md#deployment) guide, this service points to a s3proxy pod. + +The deployment specifies all regions explicitly to prevent accidental data leaks. +If one of your buckets were located in a region that's not part of the `hostAliases` key, traffic towards those buckets would not be redirected to s3proxy. +Similarly, if you want to exclude data for specific regions from going through s3proxy you can remove those regions from the deployment. + +The spec also includes a volume mount for the TLS certificate and adds it to the pod's certificate trust store. +The volume is called `ca-cert`. +The key `ca.crt` of that volume is mounted to `/etc/ssl/certs/kube-ca.crt`, which is the default certificate trust store location for that container's OpenSSL library. +Not adding the CA certificate will result in TLS authentication errors. + +3. Apply the file: `kubectl apply -f deployment-filestash.yaml` + +Afterward, you can use a port forward to access the Filestash pod: +`kubectl port-forward pod/$(kubectl get pod --selector='app=filestash' -o=jsonpath='{.items[*].metadata.name}') 8334:8334` + +4. After browsing to `localhost:8443`, Filestash will ask you to set an administrator password. +After setting it, you can directly leave the admin area by clicking the blue cloud symbol in the top left corner. +Subsequently, you can select S3 as storage backend and enter your credentials. +This will bring you to an overview of your buckets. +If you want to deploy Filestash in production, take a look at its [documentation](https://www.filestash.app/docs/). + +5. To see the logs of s3proxy intercepting requests made to S3, run: `kubectl logs -f pod/$(kubectl get pod --selector='app=s3proxy' -o=jsonpath='{.items[*].metadata.name}')` +Look out for log messages labeled `intercepting`. +There is one such log message for each message that's encrypted, decrypted, or blocked. + +6. Once you have uploaded a file with Filestash, you should be able to view the file in Filestash. +However, if you go to the AWS S3 [Web UI](https://s3.console.aws.amazon.com/s3/home) and download the file you just uploaded in Filestash, you won't be able to read it. +Another way to spot encrypted files without downloading them is to click on a file, scroll to the Metadata section, and look for the header named `x-amz-meta-constellation-encryption`. +This header holds the encrypted data encryption key of the object and is only present on objects that are encrypted by s3proxy. diff --git a/docs/versioned_docs/version-2.23/getting-started/examples/horizontal-scaling.md b/docs/versioned_docs/version-2.23/getting-started/examples/horizontal-scaling.md new file mode 100644 index 000000000..dfaf9e742 --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/examples/horizontal-scaling.md @@ -0,0 +1,98 @@ +# Horizontal Pod Autoscaling +This example demonstrates Constellation's autoscaling capabilities. It's based on the Kubernetes [HorizontalPodAutoscaler Walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/). During the following steps, Constellation will spawn new VMs on demand, verify them, add them to the cluster, and delete them again when the load has settled down. + +## Requirements +The cluster needs to be initialized with Kubernetes 1.23 or later. In addition, [autoscaling must be enabled](../../workflows/scale.md) to enable Constellation to assign new nodes dynamically. + +Just for this example specifically, the cluster should have as few worker nodes in the beginning as possible. Start with a small cluster with only *one* low-powered node for the control-plane node and *one* low-powered worker node. + +:::info +We tested the example using instances of types `Standard_DC4as_v5` on Azure and `n2d-standard-4` on GCP. +::: + +## Setup + +1. Install the Kubernetes Metrics Server: + ```bash + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` + +2. Deploy the HPA example server that's supposed to be scaled under load. + + This manifest is similar to the one from the Kubernetes HPA walkthrough, but with increased CPU limits and requests to facilitate the triggering of node scaling events. + ```bash + cat < + +Online Boutique - Web UI + + + +1. Create a namespace: + ```bash + kubectl create ns boutique + ``` +2. Deploy the application: + ```bash + kubectl apply -n boutique -f https://github.com/GoogleCloudPlatform/microservices-demo/raw/main/release/kubernetes-manifests.yaml + ``` +3. Wait for all services to become available: + ```bash + kubectl wait --for=condition=available --timeout=300s -n boutique --all deployments + ``` +4. Get the frontend's external IP address: + ```shell-session + $ kubectl get service frontend-external -n boutique | awk '{print $4}' + EXTERNAL-IP + + ``` + (`` is a placeholder for the IP assigned by your CSP.) +5. Enter the IP from the result in your browser to browse the online shop. diff --git a/docs/versioned_docs/version-2.23/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.23/getting-started/first-steps-local.md new file mode 100644 index 000000000..98f0302de --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/first-steps-local.md @@ -0,0 +1,277 @@ +# First steps with a local cluster + +A local cluster lets you deploy and test Constellation without a cloud subscription. +You have two options: + +* Use MiniConstellation to automatically deploy a two-node cluster. +* For more fine-grained control, create the cluster using the QEMU provider. + +Both options use virtualization to create a local cluster with control-plane nodes and worker nodes. They **don't** require hardware with Confidential VM (CVM) support. For attestation, they currently use a software-based vTPM provided by KVM/QEMU. + +You need an x64 machine with a Linux OS. +You can use a VM, but it needs nested virtualization. + +## Prerequisites + +* Machine requirements: + * An x86-64 CPU with at least 4 cores (6 cores are recommended) + * At least 4 GB RAM (6 GB are recommended) + * 20 GB of free disk space + * Hardware virtualization enabled in the BIOS/UEFI (often referred to as Intel VT-x or AMD-V/SVM) / nested-virtualization support when using a VM +* Software requirements: + * Linux OS with [KVM kernel module](https://www.linux-kvm.org/page/Main_Page) + * Recommended: Ubuntu 22.04 LTS + * [Docker](https://docs.docker.com/engine/install/) + * [xsltproc](https://gitlab.gnome.org/GNOME/libxslt/-/wikis/home) + * (Optional) [virsh](https://www.libvirt.org/manpages/virsh.html) to observe and access your nodes + +### Software installation on Ubuntu + +```bash +# install Docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt update +sudo apt install docker-ce +# install other dependencies +sudo apt install xsltproc +sudo snap install kubectl --classic +# install Constellation CLI +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +sudo install constellation-linux-amd64 /usr/local/bin/constellation +# do not drop forwarded packages +sudo iptables -P FORWARD ACCEPT +``` + +## Create a cluster + + + + + +With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). + + +:::caution + +MiniConstellation has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since MiniConstellation runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +The following creates your MiniConstellation cluster (may take up to 10 minutes to complete): + +```bash +constellation mini up +``` + +This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. +All `constellation` commands concerning this cluster need to be issued from this directory. + + + + +With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. + +:::caution + +Constellation on QEMU has specific soft- and hardware requirements such as a Linux OS running on an x86-64 CPU. Pay attention to all [prerequisites](#prerequisites) when setting up. + +::: + +:::note + +Since Constellation on QEMU runs on your local system, cloud features such as load balancing, +attaching persistent storage, or autoscaling aren't available. + +::: + +1. To set up your local cluster, you need to create a configuration file for Constellation first. + + ```bash + constellation config generate qemu + ``` + + This creates a [configuration file](../workflows/config.md) for QEMU called `constellation-conf.yaml`. After that, your current folder also becomes your [workspace](../architecture/orchestration.md#workspaces). All `constellation` commands for your cluster need to be executed from this directory. + +2. Now you can create your cluster and its nodes. `constellation apply` uses the options set in `constellation-conf.yaml`. + + ```bash + constellation apply -y + ``` + + The Output should look like the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type 2-vCPUs will be created. + 1 worker node of type 2-vCPUs will be created. + Creating + Cloud infrastructure created successfully. + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your setup, `constellation apply` may take 10+ minutes to complete. + + ::: + +3. Configure kubectl + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + + + +## Connect to the cluster + +Your cluster initially consists of a single control-plane node: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 66s v1.24.6 +``` + +Additional nodes will request to join the cluster shortly. Before each additional node is allowed to join the cluster, its state is verified using remote attestation by the [JoinService](../architecture/microservices.md#joinservice). +If verification passes successfully, the new node receives keys and certificates to join the cluster. + +You can follow this process by viewing the logs of the JoinService: + +```shell-session +$ kubectl logs -n kube-system daemonsets/join-service -f +{"level":"INFO","ts":"2022-10-14T09:32:20Z","caller":"cmd/main.go:48","msg":"Constellation Node Join Service","version":"2.1.0","cloudProvider":"qemu"} +{"level":"INFO","ts":"2022-10-14T09:32:20Z","logger":"validator","caller":"watcher/validator.go:96","msg":"Updating expected measurements"} +... +``` + +Once all nodes have joined your cluster, it may take a couple of minutes for all resources to become available. +You can check on the state of your cluster by running the following: + +```shell-session +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +control-plane-0 Ready control-plane 2m59s v1.24.6 +worker-0 Ready 32s v1.24.6 +``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation mini down +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +Once you are done, you can clean up the created resources using the following command: + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +This will destroy your cluster and clean up your workspace. +The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. + + + + +## Troubleshooting + +Make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### VMs have no internet access / CLI remains in "Initializing cluster" state + +`iptables` rules may prevent your VMs from accessing the internet. +Make sure your rules aren't dropping forwarded packages. + +List your rules: + +```bash +sudo iptables -S +``` + +The output may look similar to the following: + +```shell-session +-P INPUT ACCEPT +-P FORWARD DROP +-P OUTPUT ACCEPT +-N DOCKER +-N DOCKER-ISOLATION-STAGE-1 +-N DOCKER-ISOLATION-STAGE-2 +-N DOCKER-USER +``` + +If your `FORWARD` chain is set to `DROP`, you need to update your rules: + +```bash +sudo iptables -P FORWARD ACCEPT +``` diff --git a/docs/versioned_docs/version-2.23/getting-started/first-steps.md b/docs/versioned_docs/version-2.23/getting-started/first-steps.md new file mode 100644 index 000000000..fb8437a06 --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/first-steps.md @@ -0,0 +1,235 @@ +# First steps with Constellation + +The following steps guide you through the process of creating a cluster and deploying a sample app. This example assumes that you have successfully [installed and set up Constellation](install.md), +and have access to a cloud subscription. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +:::note +If you encounter any problem with the following steps, make sure to use the [latest release](https://github.com/edgelesssys/constellation/releases/latest) and check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). +::: + +## Create a cluster + +1. Create the [configuration file](../workflows/config.md) and state file for your cloud provider. If you are following the steps of this guide, there is no need to edit the file. + + + + + ```bash + constellation config generate aws + ``` + + + + + ```bash + constellation config generate azure + ``` + + + + + ```bash + constellation config generate gcp + ``` + + + + + ```bash + constellation config generate stackit + ``` + + + + +2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). + + + + + ```bash + constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config + ``` + + This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Depending on the attestation variant selected on config generation, different regions are available. + AMD SEV-SNP machines (requires the default attestation variant `awsSEVSNP`) are currently available in the following regions: + * `eu-west-1` + * `us-east-2` + + You can find a list of regions that support AMD SEV-SNP in [AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). + + NitroTPM machines (requires the attestation variant `awsNitroTPM`) are available in all regions. + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + + + + + ```bash + constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config + ``` + + This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + + + + + ```bash + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --prefix=constell-test --update-config + ``` + + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + + Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. + + + + + To use Constellation on STACKIT, the cluster will use the User Access Token (UAT) that's generated [during the install step](./install.md). + After creating the accounts, fill in the STACKIT details in `constellation-conf.yaml` under `provider.openstack`: + + * `stackitProjectID`: STACKIT project id (can be found after login on the [STACKIT portal](https://portal.stackit.cloud)) + + :::caution + + `stackitProjectID` refers to the ID of your STACKIT project. The STACKIT portal also shows the OpenStack ID that's associated with your project in some places. Make sure you insert the STACKIT project ID in the `constellation-conf.yaml` file. It's of the format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX`. + + ::: + + + + + :::tip + To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). + ::: + + + +3. Create the cluster. `constellation apply` uses options set in `constellation-conf.yaml`. + If you want to manually manage your cloud resources, for example by using [Terraform](../reference/terraform.md), follow the corresponding instructions in the [Create workflow](../workflows/create.md). + + :::tip + + On Azure, you may need to wait 15+ minutes at this point for role assignments to propagate. + + ::: + + ```bash + constellation apply -y + ``` + + This should look similar to the following: + + ```shell-session + $ constellation apply -y + Checking for infrastructure changes + The following Constellation cluster will be created: + 3 control-plane nodes of type n2d-standard-4 will be created. + 1 worker node of type n2d-standard-4 will be created. + Creating + Cloud infrastructure created successfully + Your Constellation master secret was successfully written to ./constellation-mastersecret.json + Connecting + Initializing cluster + Installing Kubernetes components + Your Constellation cluster was successfully initialized. + + Constellation cluster identifier g6iMP5wRU1b7mpOz2WEISlIYSfdAhB0oNaOg6XEwKFY= + Kubernetes configuration constellation-admin.conf + + You can now connect to your cluster by executing: + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + + The cluster's identifier will be different in your output. + Keep `constellation-mastersecret.json` somewhere safe. + This will allow you to [recover your cluster](../workflows/recovery.md) in case of a disaster. + + :::info + + Depending on your CSP and region, `constellation apply` may take 10+ minutes to complete. + + ::: + +4. Configure kubectl. + + ```bash + export KUBECONFIG="$PWD/constellation-admin.conf" + ``` + +## Deploy a sample application + +1. Deploy the [emojivoto app](https://github.com/BuoyantIO/emojivoto) + + ```bash + kubectl apply -k github.com/BuoyantIO/emojivoto/kustomize/deployment + ``` + +2. Expose the frontend service locally + + ```bash + kubectl wait --for=condition=available --timeout=60s -n emojivoto --all deployments + kubectl -n emojivoto port-forward svc/web-svc 8080:80 & + curl http://localhost:8080 + kill %1 + ``` + +## Terminate your cluster + +Use the CLI to terminate your cluster. If you manually used [Terraform](../reference/terraform.md) to manage your cloud resources, follow the corresponding instructions in the [Terminate workflow](../workflows/terminate.md). + +```bash +constellation terminate +``` + +This should give the following output: + +```shell-session +$ constellation terminate +You are about to terminate a Constellation cluster. +All of its associated resources will be DESTROYED. +This action is irreversible and ALL DATA WILL BE LOST. +Do you want to continue? [y/n]: +``` + +Confirm with `y` to terminate the cluster: + +```shell-session +Terminating ... +Your Constellation cluster was terminated successfully. +``` + +Optionally, you can also [delete your IAM resources](../workflows/config.md#deleting-an-iam-configuration). diff --git a/docs/versioned_docs/version-2.23/getting-started/install.md b/docs/versioned_docs/version-2.23/getting-started/install.md new file mode 100644 index 000000000..f072407d8 --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/install.md @@ -0,0 +1,447 @@ +# Installation and setup + +Constellation runs entirely in your cloud environment and can be controlled via a dedicated [command-line interface (CLI)](../reference/cli.md) or a [Terraform provider](../workflows/terraform-provider.md). + +## Prerequisites + +Make sure the following requirements are met: + +* Your machine is running Linux, macOS, or Windows +* You have admin rights on your machine +* [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed +* Your CSP is Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), or STACKIT + +## Install the Constellation CLI + +:::tip + +If you prefer to use Terraform, you can alternatively use the [Terraform provider](../workflows/terraform-provider.md) to manage the cluster's lifecycle. + +::: + +The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). +Install it with the following commands: + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-amd64 /usr/local/bin/constellation +``` + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-linux-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-linux-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-arm64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-arm64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/constellation-darwin-amd64 +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI to your PATH: + +```bash +sudo install constellation-darwin-amd64 /usr/local/bin/constellation +``` + + + + + +1. Download the CLI: + +```bash +Invoke-WebRequest -OutFile ./constellation.exe -Uri 'https://github.com/edgelesssys/constellation/releases/latest/download/constellation-windows-amd64.exe' +``` + +2. [Verify the signature](../workflows/verify-cli.md) (optional) + +3. Install the CLI under `C:\Program Files\Constellation\bin\constellation.exe` + +3. Add the CLI to your PATH: + + 1. Open `Advanced system settings` by searching for the App in the Windows search + 2. Go to the `Advanced` tab + 3. Click `Environment Variables…` + 4. Click variable called `Path` and click `Edit…` + 5. Click `New` + 6. Enter the path to the folder containing the binary you want on your PATH: `C:\Program Files\Constellation\bin` + + + + +:::tip +The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. +::: + +## Set up cloud credentials + +Constellation makes authenticated calls to the CSP API. Therefore, you need to set up Constellation with the credentials for your CSP. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +### Required permissions + + + + +To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreatePolicy", + "iam:CreateRole", + "iam:DeleteInstanceProfile", + "iam:DeletePolicy", + "iam:DeletePolicyVersion", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfilesForRole", + "iam:ListPolicyVersions", + "iam:ListRolePolicies", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile", + "sts:GetCallerIdentity" + ], + "Resource": "*" + } + ] +} +``` + +The built-in `AdministratorAccess` policy is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), see the permissions of [main.tf](https://github.com/edgelesssys/constellation/blob/main/terraform/infrastructure/iam/aws/main.tf). + +The built-in `PowerUserAccess` policy is a superset of these permissions. + +Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). + + + + +The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: + +* `Microsoft.Attestation` +* `Microsoft.Compute` +* `Microsoft.Insights` +* `Microsoft.ManagedIdentity` +* `Microsoft.Network` + +By default, Constellation tries to register these automatically if they haven't been registered before. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `*/register/action` \[1] +* `Microsoft.Authorization/roleAssignments/*` +* `Microsoft.Authorization/roleDefinitions/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Resources/subscriptions/resourcegroups/*` + +The built-in `Owner` role is a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `Microsoft.Attestation/attestationProviders/*` +* `Microsoft.Compute/virtualMachineScaleSets/*` +* `Microsoft.Insights/components/*` +* `Microsoft.ManagedIdentity/userAssignedIdentities/*` +* `Microsoft.Network/loadBalancers/*` +* `Microsoft.Network/loadBalancers/backendAddressPools/*` +* `Microsoft.Network/networkSecurityGroups/*` +* `Microsoft.Network/publicIPAddresses/*` +* `Microsoft.Network/virtualNetworks/*` +* `Microsoft.Network/virtualNetworks/subnets/*` +* `Microsoft.Network/natGateways/*` + +The built-in `Contributor` role is a superset of these permissions. + +Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-definitions) and [assigning roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments). + +1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. + + + + +Create a new project for Constellation or use an existing one. +Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. + +To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: + +* `iam.roles.create` +* `iam.roles.delete` +* `iam.roles.get` +* `iam.serviceAccountKeys.create` +* `iam.serviceAccountKeys.delete` +* `iam.serviceAccountKeys.get` +* `iam.serviceAccounts.create` +* `iam.serviceAccounts.delete` +* `iam.serviceAccounts.get` +* `resourcemanager.projects.getIamPolicy` +* `resourcemanager.projects.setIamPolicy` + +Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +To [create a Constellation cluster](../workflows/create.md), you need the following permissions: + +* `compute.addresses.createInternal` +* `compute.addresses.deleteInternal` +* `compute.addresses.get` +* `compute.addresses.useInternal` +* `compute.backendServices.create` +* `compute.backendServices.delete` +* `compute.backendServices.get` +* `compute.backendServices.use` +* `compute.disks.create` +* `compute.firewalls.create` +* `compute.firewalls.delete` +* `compute.firewalls.get` +* `compute.firewalls.update` +* `compute.forwardingRules.create` +* `compute.forwardingRules.delete` +* `compute.forwardingRules.get` +* `compute.forwardingRules.setLabels` +* `compute.forwardingRules.list` +* `compute.globalAddresses.create` +* `compute.globalAddresses.delete` +* `compute.globalAddresses.get` +* `compute.globalAddresses.use` +* `compute.globalForwardingRules.create` +* `compute.globalForwardingRules.delete` +* `compute.globalForwardingRules.get` +* `compute.globalForwardingRules.setLabels` +* `compute.globalOperations.get` +* `compute.healthChecks.create` +* `compute.healthChecks.delete` +* `compute.healthChecks.get` +* `compute.healthChecks.useReadOnly` +* `compute.instanceGroupManagers.create` +* `compute.instanceGroupManagers.delete` +* `compute.instanceGroupManagers.get` +* `compute.instanceGroupManagers.update` +* `compute.instanceGroups.create` +* `compute.instanceGroups.delete` +* `compute.instanceGroups.get` +* `compute.instanceGroups.update` +* `compute.instanceGroups.use` +* `compute.instances.create` +* `compute.instances.setLabels` +* `compute.instances.setMetadata` +* `compute.instances.setTags` +* `compute.instanceTemplates.create` +* `compute.instanceTemplates.delete` +* `compute.instanceTemplates.get` +* `compute.instanceTemplates.useReadOnly` +* `compute.networks.create` +* `compute.networks.delete` +* `compute.networks.get` +* `compute.networks.updatePolicy` +* `compute.routers.create` +* `compute.routers.delete` +* `compute.routers.get` +* `compute.routers.update` +* `compute.subnetworks.create` +* `compute.subnetworks.delete` +* `compute.subnetworks.get` +* `compute.subnetworks.use` +* `compute.targetTcpProxies.create` +* `compute.targetTcpProxies.delete` +* `compute.targetTcpProxies.get` +* `compute.targetTcpProxies.use` +* `iam.serviceAccounts.actAs` + +Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. + +Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). + + + + +Constellation on STACKIT requires a User Access Token (UAT) for the OpenStack API and a STACKIT service account. +The UAT already has all required permissions by default. +The STACKIT service account needs the `editor` role to create STACKIT LoadBalancers. +Look at the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) on how to create the service account and assign the role. + + + + +### Authentication + +You need to authenticate with your CSP. The following lists the required steps for *testing* and *production* environments. + +:::note +The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. +::: + + + + +**Testing** + +You can use the [AWS CloudShell](https://console.aws.amazon.com/cloudshell/home). Make sure you are [authorized to use it](https://docs.aws.amazon.com/cloudshell/latest/userguide/sec-auth-with-identities.html). + +**Production** + +Use the latest version of the [AWS CLI](https://aws.amazon.com/cli/) on a trusted machine: + +```bash +aws configure +``` + +Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). + + + + +**Testing** + +Simply open the [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). + +**Production** + +Use the latest version of the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/) on a trusted machine: + +```bash +az login +``` + +Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + + + + +**Testing** + +You can use the [Google Cloud Shell](https://cloud.google.com/shell). Make sure your [session is authorized](https://cloud.google.com/shell/docs/auth). For example, execute `gsutil` and accept the authorization prompt. + +**Production** + +Use one of the following options on a trusted machine: + +* Use the [`gcloud` CLI](https://cloud.google.com/sdk/gcloud) + + ```bash + gcloud auth application-default login + ``` + + This will ask you to log-in to your Google account and create your credentials. + The Constellation CLI will automatically load these credentials when needed. + +* Set up a service account and pass the credentials manually + + Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. + + + + +You need to authenticate with the infrastructure API (OpenStack) and create a service account (STACKIT API). + +1. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/step-1-generating-of-user-access-token-11763726.html) for obtaining a User Access Token (UAT) to use the infrastructure API +2. Create a configuration file with the credentials from the User Access Token under: + * Linux: `~/.config/openstack/clouds.yaml` + * macOS: `/Users//Library/Application Support/openstack/clouds.yaml` or `/etc/openstack/clouds.yaml` + * Windows: `%AppData%\openstack\clouds.yaml` + + + ```yaml + clouds: + stackit: + auth: + auth_url: https://keystone.api.iaas.eu01.stackit.cloud/v3 + username: REPLACE_WITH_UAT_USERNAME + password: REPLACE_WITH_UAT_PASSWORD + project_id: REPLACE_WITH_OPENSTACK_PROJECT_ID + project_name: REPLACE_WITH_STACKIT_PROJECT_NAME + user_domain_name: portal_mvp + project_domain_name: portal_mvp + region_name: RegionOne + identity_api_version: 3 + ``` + +:::caution + +`project_id` refers to the ID of your OpenStack project. The STACKIT portal also shows the STACKIT ID that's associated with your project in some places. Make sure you insert the OpenStack project ID in the `clouds.yaml` file. + +::: + +3. [Follow the STACKIT documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) for creating a service account and an access token +4. Assign the `editor` role to the service account by [following the documentation](https://docs.stackit.cloud/stackit/en/getting-started-in-service-accounts-134415831.html) +5. Create a configuration file under `~/.stackit/credentials.json` (`%USERPROFILE%\.stackit\credentials.json` on Windows) + + ```json + {"STACKIT_SERVICE_ACCOUNT_TOKEN":"REPLACE_WITH_TOKEN"} + ``` + + + + + +## Next steps + +You are now ready to [deploy your first confidential Kubernetes cluster and application](first-steps.md). diff --git a/docs/versioned_docs/version-2.23/getting-started/marketplaces.md b/docs/versioned_docs/version-2.23/getting-started/marketplaces.md new file mode 100644 index 000000000..a6763a42a --- /dev/null +++ b/docs/versioned_docs/version-2.23/getting-started/marketplaces.md @@ -0,0 +1,56 @@ +# Using Constellation via Cloud Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). + +This document explains how to run Constellation with the dynamically billed cloud marketplace images. + + + + +To use Constellation's marketplace images, ensure that you are subscribed to the [marketplace offering](https://aws.amazon.com/marketplace/pp/prodview-2mbn65nv57oys) through the web portal. + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.aws.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +Constellation has a private marketplace plan. Please [contact us](https://www.edgeless.systems/enterprise-support/) to gain access. + +To use a marketplace image, you need to accept the marketplace image's terms once for your subscription with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/vm/image/terms?view=azure-cli-latest): + +```bash +az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation +``` + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +To use a marketplace image, ensure that the account is entitled to use marketplace images by Edgeless Systems by accepting the terms through the [web portal](https://console.cloud.google.com/marketplace/vm/config/edgeless-systems-public/constellation). + +Then, enable the use of marketplace images in your Constellation `constellation-conf.yaml` [config file](../workflows/config.md): + +```bash +yq eval -i ".provider.gcp.useMarketplaceImage = true" constellation-conf.yaml +``` + + + + +On STACKIT, the selected Constellation image is always a marketplace image. You can find more information on the STACKIT portal. + + + + +Ensure that the cluster uses an official release image version (i.e., `.image=vX.Y.Z` in the `constellation-conf.yaml` file). + +From there, you can proceed with the [cluster creation](../workflows/create.md) as usual. diff --git a/docs/versioned_docs/version-2.23/intro.md b/docs/versioned_docs/version-2.23/intro.md new file mode 100644 index 000000000..0bfe86da9 --- /dev/null +++ b/docs/versioned_docs/version-2.23/intro.md @@ -0,0 +1,34 @@ +--- +slug: / +id: intro +--- +# Introduction + +Welcome to the documentation of Constellation! Constellation is a Kubernetes engine that aims to provide the best possible data security. + +![Constellation concept](/img/concept.svg) + + Constellation shields your entire Kubernetes cluster from the underlying cloud infrastructure. Everything inside is always encrypted, including at runtime in memory. For this, Constellation leverages a technology called *confidential computing* and more specifically Confidential VMs. + +:::tip +See the 📄[whitepaper](https://content.edgeless.systems/hubfs/Confidential%20Computing%20Whitepaper.pdf) for more information on confidential computing. +::: + +## Goals + +From a security perspective, Constellation is designed to keep all data always encrypted and to prevent any access from the underlying (cloud) infrastructure. This includes access from datacenter employees, privileged cloud admins, and attackers coming through the infrastructure. Such attackers could be malicious co-tenants escalating their privileges or hackers who managed to compromise a cloud server. + +From a DevOps perspective, Constellation is designed to work just like what you would expect from a modern Kubernetes engine. + +## Use cases + +Constellation provides unique security [features](overview/confidential-kubernetes.md) and [benefits](overview/security-benefits.md). The core use cases are: + +* Increasing the overall security of your clusters +* Increasing the trustworthiness of your SaaS offerings +* Moving sensitive workloads from on-prem to the cloud +* Meeting regulatory requirements + +## Next steps + +You can learn more about the concept of Confidential Kubernetes, features, security benefits, and performance of Constellation in the *Basics* section. To jump right into the action head to *Getting started*. diff --git a/docs/versioned_docs/version-2.23/overview/clouds.md b/docs/versioned_docs/version-2.23/overview/clouds.md new file mode 100644 index 000000000..b2695d28e --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/clouds.md @@ -0,0 +1,66 @@ +# Feature status of clouds + +What works on which cloud? Currently, Confidential VMs (CVMs) are available in varying quality on the different clouds and software stacks. + +For Constellation, the ideal environment provides the following: + +1. Ability to run arbitrary software and images inside CVMs +2. CVMs based on AMD SEV-SNP (available in EPYC CPUs since the Milan generation) or Intel TDX (available in Xeon CPUs since the Sapphire Rapids generation) +3. Ability for CVM guests to obtain raw hardware attestation statements +4. Reviewable, open-source firmware inside CVMs +5. Capability of the firmware to attest the integrity of the code it passes control to, e.g., with an embedded virtual TPM (vTPM) + +(1) is a functional must-have. (2)--(5) are required for remote attestation that fully keeps the infrastructure/cloud out. Constellation can work without them or with approximations, but won't protect against certain privileged attackers anymore. + +The following table summarizes the state of features for different infrastructures. + +| **Feature** | **AWS** | **Azure** | **GCP** | **STACKIT** | **OpenStack (Yoga)** | +|-----------------------------------|---------|-----------|---------|--------------|----------------------| +| **1. Custom images** | Yes | Yes | Yes | Yes | Yes | +| **2. SEV-SNP or TDX** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **3. Raw guest attestation** | Yes | Yes | Yes | No | Depends on kernel/HV | +| **4. Reviewable firmware** | Yes | No* | No | No | Depends on kernel/HV | +| **5. Confidential measured boot** | No | Yes | No | No | Depends on kernel/HV | + +## Amazon Web Services (AWS) + +Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). +Regarding (3), AWS provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the [firmware is open source](https://github.com/aws/uefi) and can be reproducibly built. + +## Microsoft Azure + +With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. +Regarding (3), Azure provides direct access to attestation statements. +The firmware runs in an isolated domain inside the CVM and exposes a vTPM (5), but it's closed source (4). +On SEV-SNP, Azure uses VM Privilege Level (VMPL) isolation for the separation of firmware and the rest of the VM; on TDX, they use TD partitioning. +This firmware is signed by Azure. +The signature is reflected in the attestation statements of CVMs. +Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). + +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. + +## Google Cloud Platform (GCP) + +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#technologies) are based on AMD SEV-ES or SEV-SNP. +Regarding (3), with their SEV-SNP offering Google provides direct access to attestation statements. +However, regarding (5), attestation is partially based on the [Shielded VM vTPM](https://cloud.google.com/compute/shielded-vm/docs/shielded-vm#vtpm) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by Google's hypervisor. +Hence, the hypervisor is currently part of Constellation's TCB. +Regarding (4), the CVMs still include closed-source firmware. + +[TDX on Google](https://cloud.google.com/blog/products/identity-security/confidential-vms-on-intel-cpus-your-datas-new-intelligent-defense) is in public preview. +With it, Constellation would have a similar TCB and attestation flow as with the current SEV-SNP offering. + +## STACKIT + +[STACKIT Compute Engine](https://www.stackit.de/en/product/stackit-compute-engine/) supports AMD SEV-ES. A vTPM is used for measured boot, which is a vTPM managed by STACKIT's hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. + +## OpenStack + +OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. + +## Conclusion + +The different clouds and software like the Linux kernel and OpenStack are in the process of building out their support for state-of-the-art CVMs. Azure has already most features in place. For Constellation, the status quo means that the TCB has different shapes on different infrastructures. With broad SEV-SNP support coming to the Linux kernel, we soon expect a normalization of features across infrastructures. diff --git a/docs/versioned_docs/version-2.23/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.23/overview/confidential-kubernetes.md new file mode 100644 index 000000000..bff8c3322 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/confidential-kubernetes.md @@ -0,0 +1,42 @@ +# Confidential Kubernetes + +We use the term *Confidential Kubernetes* to refer to the concept of using confidential-computing technology to shield entire Kubernetes clusters from the infrastructure. The three defining properties of this concept are: + +1. **Workload shielding**: the confidentiality and integrity of all workload-related data and code are enforced. +2. **Control plane shielding**: the confidentiality and integrity of the cluster's control plane, state, and workload configuration are enforced. +3. **Attestation and verifiability**: the two properties above can be verified remotely based on hardware-rooted cryptographic certificates. + +Each of the above properties is equally important. Only with all three in conjunction, an entire cluster can be shielded without gaps. + +## Constellation security features + +Constellation implements the Confidential Kubernetes concept with the following security features. + +* **Runtime encryption**: Constellation runs all Kubernetes nodes inside Confidential VMs (CVMs). This gives runtime encryption for the entire cluster. +* **Network and storage encryption**: Constellation augments this with transparent encryption of the [network](../architecture/networking.md), [persistent storage](../architecture/encrypted-storage.md), and other managed storage like [AWS S3](../architecture/encrypted-storage.md#encrypted-s3-object-storage). Thus, workloads and control plane are truly end-to-end encrypted: at rest, in transit, and at runtime. +* **Transparent key management**: Constellation manages the corresponding [cryptographic keys](../architecture/keys.md) inside CVMs. +* **Node attestation and verification**: Constellation verifies the integrity of each new CVM-based node using [remote attestation](../architecture/attestation.md). Only "good" nodes receive the cryptographic keys required to access the network and storage of a cluster. +* **Confidential computing-optimized images**: A node is "good" if it's running a signed Constellation [node image](../architecture/images.md) inside a CVM and is in the expected state. (Node images are hardware-measured during boot. The measurements are reflected in the attestation statements that are produced by nodes and verified by Constellation.) +* **"Whole cluster" attestation**: Towards the DevOps engineer, Constellation provides a single hardware-rooted certificate from which all of the above can be verified. + +With the above, Constellation wraps an entire cluster into one coherent and verifiable *confidential context*. The concept is depicted in the following. + +![Confidential Kubernetes](../_media/concept-constellation.svg) + +## Comparison: Managed Kubernetes with CVMs + +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. + +![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) + +The following table highlights the key differences in terms of features. + +| | Managed Kubernetes with CVMs | Confidential Kubernetes (Constellation✨) | +|-------------------------------------|------------------------------|--------------------------------------------| +| Runtime encryption | Partial (data plane only)| **Yes** | +| Node image verification | No | **Yes** | +| Full cluster attestation | No | **Yes** | +| Transparent network encryption | No | **Yes** | +| Transparent storage encryption | No | **Yes** | +| Confidential key management | No | **Yes** | +| Cloud agnostic / multi-cloud | No | **Yes** | diff --git a/docs/versioned_docs/version-2.23/overview/license.md b/docs/versioned_docs/version-2.23/overview/license.md new file mode 100644 index 000000000..98a9cbf94 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/license.md @@ -0,0 +1,15 @@ +# License + +Constellation is available under the [Business Source License 1.1](https://github.com/edgelesssys/constellation/blob/main/LICENSE). + +You may use it free of charge for non-production use ("Community License"). + +## Enterprise License + +Enterprise Licenses permit production use and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/). + +Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`. + +## CSP Marketplaces + +Constellation is available through the Marketplaces of AWS, Azure, GCP, and STACKIT. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your CSP account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/). diff --git a/docs/versioned_docs/version-2.23/overview/performance/application.md b/docs/versioned_docs/version-2.23/overview/performance/application.md new file mode 100644 index 000000000..c67d59644 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/performance/application.md @@ -0,0 +1,102 @@ +# Application benchmarks + +## HashiCorp Vault + +[HashiCorp Vault](https://www.vaultproject.io/) is a distributed secrets management software that can be deployed to Kubernetes. +HashiCorp maintains a benchmarking tool for vault, [vault-benchmark](https://github.com/hashicorp/vault-benchmark/). +Vault-benchmark generates load on a Vault deployment and measures response times. + +This article describes the results from running vault-benchmark on Constellation, AKS, and GKE. +You can find the setup for producing the data discussed in this article in the [vault-benchmarks](https://github.com/edgelesssys/vault-benchmarks) repository. + +The Vault API used during benchmarking is the [transits secret engine](https://developer.hashicorp.com/vault/docs/secrets/transit). +This allows services to send data to Vault for encryption, decryption, signing, and verification. + +## Results + +On each run, vault-benchmark sends requests and measures the latencies. +The measured latencies are aggregated through various statistical features. +After running the benchmark n times, the arithmetic mean over a subset of the reported statistics is calculated. +The selected features are arithmetic mean, 99th percentile, minimum, and maximum. + +Arithmetic mean gives a general sense of the latency on each target. +The 99th percentile shows performance in (most likely) erroneous states. +Minimum and maximum mark the range within which latency varies each run. + +The benchmark was configured with 1300 workers and 10 seconds per run. +Those numbers were chosen empirically. +The latency was stabilizing at 10 seconds runtime, not changing with further increase. +Increasing the number of workers beyond 1300 leads to request failures, marking the limit Vault was able to handle in this setup. +All results are based on 100 runs. + +The following data was generated while running five replicas, one primary, and four standby nodes. +All numbers are in seconds if not indicated otherwise. +``` +========== Results AKS ========== +Mean: mean: 1.632200, variance: 0.002057 +P99: mean: 5.480679, variance: 2.263700 +Max: mean: 6.651001, variance: 2.808401 +Min: mean: 0.011415, variance: 0.000133 +========== Results GKE ========== +Mean: mean: 1.656435, variance: 0.003615 +P99: mean: 6.030807, variance: 3.955051 +Max: mean: 7.164843, variance: 3.300004 +Min: mean: 0.010233, variance: 0.000111 +========== Results C11n ========== +Mean: mean: 1.651549, variance: 0.001610 +P99: mean: 5.780422, variance: 3.016106 +Max: mean: 6.942997, variance: 3.075796 +Min: mean: 0.013774, variance: 0.000228 +========== AKS vs C11n ========== +Mean: +1.171577 % (AKS is faster) +P99: +5.185495 % (AKS is faster) +Max: +4.205618 % (AKS is faster) +Min: +17.128781 % (AKS is faster) +========== GKE vs C11n ========== +Mean: -0.295851 % (GKE is slower) +P99: -4.331603 % (GKE is slower) +Max: -3.195248 % (GKE is slower) +Min: +25.710886 % (GKE is faster) +``` + +**Interpretation**: Latencies are all within ~5% of each other. +AKS performs slightly better than GKE and Constellation (C11n) in all cases except minimum latency. +Minimum latency is the lowest for GKE. +Compared to GKE, Constellation had slightly lower peak latencies (99th percentile and maximum), indicating that Constellation could have handled slightly more concurrent accesses than GKE. +Overall, performance is at comparable levels across all three distributions. +Based on these numbers, you can use a similarly sized Constellation cluster to run your existing Vault deployment. + +### Visualization + +The following plots visualize the data presented above as [box plots](https://en.wikipedia.org/wiki/Box_plot). +The whiskers denote the minimum and maximum. +The box stretches from the 25th to the 75th percentile, with the dividing bar marking the 50th percentile. +The circles outside the whiskers denote outliers. + +
+Mean Latency + +![Mean Latency](../../_media/benchmark_vault/5replicas/mean_latency.png) + +
+ +
+99th Percentile Latency + +![99th Percentile Latency](../../_media/benchmark_vault/5replicas/p99_latency.png) + +
+ +
+Maximum Latency + +![Maximum Latency](../../_media/benchmark_vault/5replicas/max_latency.png) + +
+ +
+Minimum Latency + +![Minimum Latency](../../_media/benchmark_vault/5replicas/min_latency.png) + +
diff --git a/docs/versioned_docs/version-2.23/overview/performance/compute.md b/docs/versioned_docs/version-2.23/overview/performance/compute.md new file mode 100644 index 000000000..88dd4b1b2 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/performance/compute.md @@ -0,0 +1,11 @@ +# Impact of runtime encryption on compute performance + +All nodes in a Constellation cluster are executed inside Confidential VMs (CVMs). Consequently, the performance of Constellation is inherently linked to the performance of these CVMs. + +## AMD and Azure benchmarking + +AMD and Azure have collectively released a [performance benchmark](https://community.amd.com/t5/business/microsoft-azure-confidential-computing-powered-by-3rd-gen-epyc/ba-p/497796) for CVMs that utilize 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. This benchmark, which included a variety of mostly compute-intensive tests such as SPEC CPU 2017 and CoreMark, demonstrated that CVMs experience only minor performance degradation (ranging from 2% to 8%) when compared to standard VMs. Such results are indicative of the performance that can be expected from compute-intensive workloads running with Constellation on Azure. + +## AMD and Google benchmarking + +Similarly, AMD and Google have jointly released a [performance benchmark](https://www.amd.com/system/files/documents/3rd-gen-epyc-gcp-c2d-conf-compute-perf-brief.pdf) for CVMs employing 3rd Gen AMD EPYC processors (Milan) with SEV-SNP. With high-performance computing workloads such as WRF, NAMD, Ansys CFS, and Ansys LS_DYNA, they observed analogous findings, with only minor performance degradation (between 2% and 4%) compared to standard VMs. These outcomes are reflective of the performance that can be expected for compute-intensive workloads running with Constellation on GCP. diff --git a/docs/versioned_docs/version-2.23/overview/performance/io.md b/docs/versioned_docs/version-2.23/overview/performance/io.md new file mode 100644 index 000000000..3ae796f8a --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/performance/io.md @@ -0,0 +1,204 @@ +# I/O performance benchmarks + +To assess the overall performance of Constellation, this benchmark evaluates Constellation v2.6.0 in terms of storage I/O using [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) and network performance using the [Kubernetes Network Benchmark](https://github.com/InfraBuilder/k8s-bench-suite#knb--kubernetes-network-be). + +This benchmark tested Constellation on Azure and GCP and compared the results against the managed Kubernetes offerings AKS and GKE. + +## Configurations + +### Constellation + +The benchmark was conducted with Constellation v2.6.0, Kubernetes v1.25.7, and Cilium v1.12. +It ran on the following infrastructure configurations. + +Constellation on Azure: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `DC4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `true` +- Region: `West US` +- Zone: `2` + +Constellation on GCP: + +- Nodes: 3 (1 Control-plane, 2 Worker) +- Machines: `n2d-standard-4`: 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `true` +- Zone: `europe-west3-b` + +### AKS + +On AKS, the benchmark used Kubernetes `v1.24.9` and nodes with version `AKSUbuntu-1804gen2containerd-2023.02.15`. +AKS ran with the [`kubenet`](https://learn.microsoft.com/en-us/azure/aks/concepts-network#kubenet-basic-networking) CNI and the [default CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-disk-csi) for Azure Disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `D4as_v5`: 3rd Generation AMD EPYC 7763v (Milan) processor with 4 Cores, 16 GiB memory +- CVM: `false` +- Region: `West US` +- Zone: `2` + +### GKE + +On GKE, the benchmark used Kubernetes `v1.24.9` and nodes with version `1.24.9-gke.3200`. +GKE ran with the [`kubenet`](https://cloud.google.com/kubernetes-engine/docs/concepts/network-overview) CNI and the [default CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) for Compute Engine persistent disk. + +The following infrastructure configurations was used: + +- Nodes: 2 (2 Worker) +- Machines: `n2d-standard-4` 2nd Generation AMD EPYC (Rome) processor with 4 Cores, 16 GiB of memory +- CVM: `false` +- Zone: `europe-west3-b` + +## Results + +### Network + +This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. +The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). + +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). +The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). +Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. + +Constellation on Azure and AKS used an MTU of 1500. +Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. + +The difference in network bandwidth can largely be attributed to two factors. + +- Constellation's [network encryption](../../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. + +#### Pod-to-Pod + +In this scenario, the client Pod connects directly to the server pod via its IP address. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] + end + subgraph Node B + Server[Server] + end + Client ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2Pod Azure benchmark graph](../../_media/benchmark_net_p2p_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2Pod GCP benchmark graph](../../_media/benchmark_net_p2p_gcp.png) + +#### Pod-to-Service + +In this scenario, the client Pod connects to the server Pod via a ClusterIP service. This is more relevant to real-world use cases. + +```mermaid +flowchart LR + subgraph Node A + Client[Client] ==>|traffic| Service[Service] + end + subgraph Node B + Server[Server] + end + Service ==>|traffic| Server +``` + +The results for "Pod-to-Pod" on Azure are as follows: + +![Network Pod2SVC Azure benchmark graph](../../_media/benchmark_net_p2svc_azure.png) + +The results for "Pod-to-Pod" on GCP are as follows: + +![Network Pod2SVC GCP benchmark graph](../../_media/benchmark_net_p2svc_gcp.png) + +In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. + +Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + +### Storage I/O + +Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). +Upon requesting persistent storage through a PVC, GKE and AKS will provision a PV as defined by a default [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/). +Constellation provides persistent storage on Azure and GCP [that's encrypted on the CSI layer](../../architecture/encrypted-storage.md). +Similarly, upon a PVC request, Constellation will provision a PV via a default storage class. + +For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. +The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + +- 6400 (20000 burst) IOPS +- 144 MB/s (600 MB/s burst) throughput + +However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + +- 500 (600 burst) IOPS +- 60 MB/s (150 MB/s burst) throughput + +For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. +The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + +- 3,000 read IOPS +- 15,000 write IOPS +- 240 MB/s read throughput +- 240 MB/s write throughput + +However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + +- 2400 read IOPS +- 2400 write IOPS +- 112 MB/s read throughput +- 112 MB/s write throughput + +The [`fio`](https://fio.readthedocs.io/en/latest/fio_doc.html) benchmark consists of several tests. +The benchmark used [`Kubestr`](https://github.com/kastenhq/kubestr) to run `fio` in Kubernetes. +The default test performs randomized access patterns that accurately depict worst-case I/O scenarios for most applications. + +The following `fio` settings were used: + +- No Cloud caching +- No OS caching +- Single CPU +- 60 seconds runtime +- 10 seconds ramp-up time +- 10 GiB file +- IOPS: 4 KB blocks and 128 iodepth +- Bandwidth: 1024 KB blocks and 128 iodepth + +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). + +The results for IOPS on Azure are as follows: + +![I/O IOPS Azure benchmark graph](../../_media/benchmark_fio_azure_iops.png) + +The results for IOPS on GCP are as follows: + +![I/O IOPS GCP benchmark graph](../../_media/benchmark_fio_gcp_iops.png) + +The results for bandwidth on Azure are as follows: + +![I/O bandwidth Azure benchmark graph](../../_media/benchmark_fio_azure_bw.png) + +The results for bandwidth on GCP are as follows: + +![I/O bandwidth GCP benchmark graph](../../_media/benchmark_fio_gcp_bw.png) + +On GCP, the results exceed the maximum performance guarantees of the chosen disk type. There are two possible explanations for this. The first is that there may be cloud caching in place that isn't configurable. Alternatively, the underlying provisioned disk size may be larger than what was requested, resulting in higher performance boundaries. + +When comparing Constellation on GCP with GKE, Constellation has similar bandwidth but about 10% less IOPS performance. On Azure, Constellation has similar IOPS performance compared to AKS, where both likely hit the maximum storage performance. However, Constellation has approximately 15% less read and write bandwidth. + +## Conclusion + +Despite the added [security benefits](../security-benefits.md) that Constellation provides, it only incurs a slight performance overhead when compared to managed Kubernetes offerings such as AKS and GKE. In most compute benchmarks, Constellation is on par with it's alternatives. +While it may be slightly slower in certain I/O scenarios due to network and storage encryption, there is ongoing work to reduce this overhead to single digits. + +For instance, storage encryption only adds between 10% to 15% overhead in terms of bandwidth and IOPS. +Meanwhile, the biggest performance impact that Constellation currently faces is network encryption, which can incur up to 58% overhead on a 10 Gbps network. +However, the Cilium team has conducted [benchmarks with Cilium using WireGuard encryption](https://docs.cilium.io/en/latest/operations/performance/benchmark/#encryption-wireguard-ipsec) on a 100 Gbps network that yielded over 15 Gbps. +We're confident that Constellation will provide a similar level of performance with an upcoming release. + +Overall, Constellation strikes a great balance between security and performance, and we're continuously working to improve its performance capabilities while maintaining its high level of security. diff --git a/docs/versioned_docs/version-2.23/overview/performance/performance.md b/docs/versioned_docs/version-2.23/overview/performance/performance.md new file mode 100644 index 000000000..59bf86602 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/performance/performance.md @@ -0,0 +1,17 @@ +# Performance analysis of Constellation + +This section provides a comprehensive examination of the performance characteristics of Constellation. + +## Runtime encryption + +Runtime encryption affects compute performance. [Benchmarks by Azure and Google](compute.md) show that the performance degradation of Confidential VMs (CVMs) is small, ranging from 2% to 8% for compute-intensive workloads. + +## I/O performance benchmarks + +We evaluated the [I/O performance](io.md) of Constellation, utilizing a collection of synthetic benchmarks targeting networking and storage. +We further compared this performance to native managed Kubernetes offerings from various cloud providers, to better understand how Constellation stands in relation to standard practices. + +## Application benchmarking + +To gauge Constellation's applicability to well-known applications, we performed a [benchmark of HashiCorp Vault](application.md) running on Constellation. +The results were then compared to deployments on the managed Kubernetes offerings from different cloud providers, providing a tangible perspective on Constellation's performance in actual deployment scenarios. diff --git a/docs/versioned_docs/version-2.23/overview/product.md b/docs/versioned_docs/version-2.23/overview/product.md new file mode 100644 index 000000000..4b5d90706 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/product.md @@ -0,0 +1,12 @@ +# Product features + +Constellation is a Kubernetes engine that aims to provide the best possible data security in combination with enterprise-grade scalability and reliability features---and a smooth user experience. + +From a security perspective, Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and corresponding security features, which shield your entire cluster from the underlying infrastructure. + +From an operational perspective, Constellation provides the following key features: + +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, Google Cloud Platform (GCP), and STACKIT. Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. +* **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. +* **Support for Terraform**: Constellation includes a [Terraform provider](../workflows/terraform-provider.md) that lets you manage the full lifecycle of your cluster via Terraform. diff --git a/docs/versioned_docs/version-2.23/overview/security-benefits.md b/docs/versioned_docs/version-2.23/overview/security-benefits.md new file mode 100644 index 000000000..51a8b64f5 --- /dev/null +++ b/docs/versioned_docs/version-2.23/overview/security-benefits.md @@ -0,0 +1,22 @@ +# Security benefits and threat model + +Constellation implements the [Confidential Kubernetes](confidential-kubernetes.md) concept and shields entire Kubernetes deployments from the infrastructure. More concretely, Constellation decreases the size of the trusted computing base (TCB) of a Kubernetes deployment. The TCB is the totality of elements in a computing environment that must be trusted not to be compromised. A smaller TCB results in a smaller attack surface. The following diagram shows how Constellation removes the *cloud & datacenter infrastructure* and the *physical hosts*, including the hypervisor, the host OS, and other components, from the TCB (red). Inside the confidential context (green), Kubernetes remains part of the TCB, but its integrity is attested and can be [verified](../workflows/verify-cluster.md). + +![TCB comparison](../_media/tcb.svg) + +Given this background, the following describes the concrete threat classes that Constellation addresses. + +## Insider access + +Employees and third-party contractors of cloud service providers (CSPs) have access to different layers of the cloud infrastructure. +This opens up a large attack surface where workloads and data can be read, copied, or manipulated. With Constellation, Kubernetes deployments are shielded from the infrastructure and thus such accesses are prevented. + +## Infrastructure-based attacks + +Malicious cloud users ("hackers") may break out of their tenancy and access other tenants' data. Advanced attackers may even be able to establish a permanent foothold within the infrastructure and access data over a longer period. Analogously to the *insider access* scenario, Constellation also prevents access to a deployment's data in this scenario. + +## Supply chain attacks + +Supply chain security is receiving lots of attention recently due to an [increasing number of recorded attacks](https://www.enisa.europa.eu/news/enisa-news/understanding-the-increase-in-supply-chain-security-attacks). For instance, a malicious actor could attempt to tamper Constellation node images (including Kubernetes and other software) before they're loaded in the confidential VMs of a cluster. Constellation uses [remote attestation](../architecture/attestation.md) in conjunction with public [transparency logs](../workflows/verify-cli.md) to prevent this. + +In the future, Constellation will extend this feature to customer workloads. This will enable cluster owners to create auditable policies that precisely define which containers can run in a given deployment. diff --git a/docs/versioned_docs/version-2.23/reference/cli.md b/docs/versioned_docs/version-2.23/reference/cli.md new file mode 100644 index 000000000..7cbc0be8d --- /dev/null +++ b/docs/versioned_docs/version-2.23/reference/cli.md @@ -0,0 +1,873 @@ +# CLI reference + + + +Use the Constellation CLI to create and manage your clusters. + +Usage: + +``` +constellation [command] +``` +Commands: + +* [config](#constellation-config): Work with the Constellation configuration file + * [generate](#constellation-config-generate): Generate a default configuration and state file + * [fetch-measurements](#constellation-config-fetch-measurements): Fetch measurements for configured cloud provider and image + * [instance-types](#constellation-config-instance-types): Print the supported instance types for all cloud providers + * [kubernetes-versions](#constellation-config-kubernetes-versions): Print the Kubernetes versions supported by this CLI + * [migrate](#constellation-config-migrate): Migrate a configuration file to a new version +* [create](#constellation-create): Create instances on a cloud platform for your Constellation cluster +* [apply](#constellation-apply): Apply a configuration to a Constellation cluster +* [mini](#constellation-mini): Manage MiniConstellation clusters + * [up](#constellation-mini-up): Create and initialize a new MiniConstellation cluster + * [down](#constellation-mini-down): Destroy a MiniConstellation cluster +* [status](#constellation-status): Show status of a Constellation cluster +* [verify](#constellation-verify): Verify the confidential properties of a Constellation cluster +* [upgrade](#constellation-upgrade): Find and apply upgrades to your Constellation cluster + * [check](#constellation-upgrade-check): Check for possible upgrades + * [apply](#constellation-upgrade-apply): Apply an upgrade to a Constellation cluster +* [recover](#constellation-recover): Recover a completely stopped Constellation cluster +* [terminate](#constellation-terminate): Terminate a Constellation cluster +* [iam](#constellation-iam): Work with the IAM configuration on your cloud provider + * [create](#constellation-iam-create): Create IAM configuration on a cloud platform for your Constellation cluster + * [aws](#constellation-iam-create-aws): Create IAM configuration on AWS for your Constellation cluster + * [azure](#constellation-iam-create-azure): Create IAM configuration on Microsoft Azure for your Constellation cluster + * [gcp](#constellation-iam-create-gcp): Create IAM configuration on GCP for your Constellation cluster + * [destroy](#constellation-iam-destroy): Destroy an IAM configuration and delete local Terraform files + * [upgrade](#constellation-iam-upgrade): Find and apply upgrades to your IAM profile + * [apply](#constellation-iam-upgrade-apply): Apply an upgrade to an IAM profile +* [version](#constellation-version): Display version of this CLI +* [init](#constellation-init): Initialize the Constellation cluster +* [ssh](#constellation-ssh): Generate a certificate for emergency SSH access + +## constellation config + +Work with the Constellation configuration file + +### Synopsis + +Work with the Constellation configuration file. + +### Options + +``` + -h, --help help for config +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config generate + +Generate a default configuration and state file + +### Synopsis + +Generate a default configuration and state file for your selected cloud provider. + +``` +constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] +``` + +### Options + +``` + -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used + -h, --help help for generate + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.30") + -t, --tags strings additional tags for created resources given a list of key=value +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config fetch-measurements + +Fetch measurements for configured cloud provider and image + +### Synopsis + +Fetch measurements for configured cloud provider and image. + +A config needs to be generated first. + +``` +constellation config fetch-measurements [flags] +``` + +### Options + +``` + -h, --help help for fetch-measurements + -s, --signature-url string alternative URL to fetch measurements' signature from + -u, --url string alternative URL to fetch measurements from +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config instance-types + +Print the supported instance types for all cloud providers + +### Synopsis + +Print the supported instance types for all cloud providers. + +``` +constellation config instance-types [flags] +``` + +### Options + +``` + -h, --help help for instance-types +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config kubernetes-versions + +Print the Kubernetes versions supported by this CLI + +### Synopsis + +Print the Kubernetes versions supported by this CLI. + +``` +constellation config kubernetes-versions [flags] +``` + +### Options + +``` + -h, --help help for kubernetes-versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation config migrate + +Migrate a configuration file to a new version + +### Synopsis + +Migrate a configuration file to a new version. + +``` +constellation config migrate [flags] +``` + +### Options + +``` + -h, --help help for migrate +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation create + +Create instances on a cloud platform for your Constellation cluster + +### Synopsis + +Create instances on a cloud platform for your Constellation cluster. + +``` +constellation create [flags] +``` + +### Options + +``` + -h, --help help for create + -y, --yes create the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation apply + +Apply a configuration to a Constellation cluster + +### Synopsis + +Apply a configuration to a Constellation cluster to initialize or upgrade the cluster. + +``` +constellation apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | init | attestationconfig | certsans | helm | image | k8s } + -y, --yes run command without further confirmation + WARNING: the command might delete or update existing resources without additional checks. Please read the docs. + +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini + +Manage MiniConstellation clusters + +### Synopsis + +Manage MiniConstellation clusters. + +### Options + +``` + -h, --help help for mini +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini up + +Create and initialize a new MiniConstellation cluster + +### Synopsis + +Create and initialize a new MiniConstellation cluster. + +A mini cluster consists of a single control-plane and worker node, hosted using QEMU/KVM. + +``` +constellation mini up [flags] +``` + +### Options + +``` + -h, --help help for up + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config (default true) +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation mini down + +Destroy a MiniConstellation cluster + +### Synopsis + +Destroy a MiniConstellation cluster. + +``` +constellation mini down [flags] +``` + +### Options + +``` + -h, --help help for down + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation status + +Show status of a Constellation cluster + +### Synopsis + +Show the status of a constellation cluster. + +Shows microservice, image, and Kubernetes versions installed in the cluster. Also shows status of current version upgrades. + +``` +constellation status [flags] +``` + +### Options + +``` + -h, --help help for status +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation verify + +Verify the confidential properties of a Constellation cluster + +### Synopsis + +Verify the confidential properties of a Constellation cluster. +If arguments aren't specified, values are read from `constellation-state.yaml`. + +``` +constellation verify [flags] +``` + +### Options + +``` + --cluster-id string expected cluster identifier + -h, --help help for verify + -e, --node-endpoint string endpoint of the node to verify, passed as HOST[:PORT] + -o, --output string print the attestation document in the output format {json|raw} +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade + +Find and apply upgrades to your Constellation cluster + +### Synopsis + +Find and apply upgrades to your Constellation cluster. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade check + +Check for possible upgrades + +### Synopsis + +Check which upgrades can be applied to your Constellation Cluster. + +``` +constellation upgrade check [flags] +``` + +### Options + +``` + -h, --help help for check + --ref string the reference to use for querying new versions (default "-") + --stream string the stream to use for querying new versions (default "stable") + -u, --update-config update the specified config file with the suggested versions +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation upgrade apply + +Apply an upgrade to a Constellation cluster + +### Synopsis + +Apply an upgrade to a Constellation cluster by applying the chosen configuration. + +``` +constellation upgrade apply [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for apply + --skip-helm-wait install helm charts without waiting for deployments to be ready + --skip-phases strings comma-separated list of upgrade phases to skip + one or multiple of { infrastructure | helm | image | k8s } + -y, --yes run upgrades without further confirmation + WARNING: might delete your resources in case you are using cert-manager in your cluster. Please read the docs. + WARNING: might unintentionally overwrite measurements in the running cluster. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation recover + +Recover a completely stopped Constellation cluster + +### Synopsis + +Recover a Constellation cluster by sending a recovery key to an instance in the boot stage. + +This is only required if instances restart without other instances available for bootstrapping. + +``` +constellation recover [flags] +``` + +### Options + +``` + -e, --endpoint string endpoint of the instance, passed as HOST[:PORT] + -h, --help help for recover +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation terminate + +Terminate a Constellation cluster + +### Synopsis + +Terminate a Constellation cluster. + +The cluster can't be started again, and all persistent storage will be lost. + +``` +constellation terminate [flags] +``` + +### Options + +``` + -h, --help help for terminate + -y, --yes terminate the cluster without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam + +Work with the IAM configuration on your cloud provider + +### Synopsis + +Work with the IAM configuration on your cloud provider. + +### Options + +``` + -h, --help help for iam +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create + +Create IAM configuration on a cloud platform for your Constellation cluster + +### Synopsis + +Create IAM configuration on a cloud platform for your Constellation cluster. + +### Options + +``` + -h, --help help for create + --update-config update the config file with the specific IAM information + -y, --yes create the IAM configuration without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam create aws + +Create IAM configuration on AWS for your Constellation cluster + +### Synopsis + +Create IAM configuration on AWS for your Constellation cluster. + +``` +constellation iam create aws [flags] +``` + +### Options + +``` + -h, --help help for aws + --prefix string name prefix for all resources (required) + --zone string AWS availability zone the resources will be created in, e.g., us-east-2a (required) + See the Constellation docs for a list of currently supported regions. +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create azure + +Create IAM configuration on Microsoft Azure for your Constellation cluster + +### Synopsis + +Create IAM configuration on Microsoft Azure for your Constellation cluster. + +``` +constellation iam create azure [flags] +``` + +### Options + +``` + -h, --help help for azure + --region string region the resources will be created in, e.g., westus (required) + --resourceGroup string name prefix of the two resource groups your cluster / IAM resources will be created in (required) + --servicePrincipal string name of the service principal that will be created (required) + --subscriptionID string subscription ID of the Azure account. Required if the 'ARM_SUBSCRIPTION_ID' environment variable is not set +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam create gcp + +Create IAM configuration on GCP for your Constellation cluster + +### Synopsis + +Create IAM configuration on GCP for your Constellation cluster. + +``` +constellation iam create gcp [flags] +``` + +### Options + +``` + -h, --help help for gcp + --prefix string Prefix for the service account ID and VM ID that will be created (required) + Must be letters, digits, or hyphens. + --projectID string ID of the GCP project the configuration will be created in (required) + Find it on the welcome screen of your project: https://console.cloud.google.com/welcome + --zone string GCP zone the cluster will be deployed in (required) + Find a list of available zones here: https://cloud.google.com/compute/docs/regions-zones#available +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + --update-config update the config file with the specific IAM information + -C, --workspace string path to the Constellation workspace + -y, --yes create the IAM configuration without further confirmation +``` + +## constellation iam destroy + +Destroy an IAM configuration and delete local Terraform files + +### Synopsis + +Destroy an IAM configuration and delete local Terraform files. + +``` +constellation iam destroy [flags] +``` + +### Options + +``` + -h, --help help for destroy + -y, --yes destroy the IAM configuration without asking for confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade + +Find and apply upgrades to your IAM profile + +### Synopsis + +Find and apply upgrades to your IAM profile. + +### Options + +``` + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation iam upgrade apply + +Apply an upgrade to an IAM profile + +### Synopsis + +Apply an upgrade to an IAM profile. + +``` +constellation iam upgrade apply [flags] +``` + +### Options + +``` + -h, --help help for apply + -y, --yes run upgrades without further confirmation +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation version + +Display version of this CLI + +### Synopsis + +Display version of this CLI. + +``` +constellation version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation init + +Initialize the Constellation cluster + +### Synopsis + +Initialize the Constellation cluster. + +Start your confidential Kubernetes. + +``` +constellation init [flags] +``` + +### Options + +``` + --conformance enable conformance mode + -h, --help help for init + --merge-kubeconfig merge Constellation kubeconfig file with default kubeconfig file in $HOME/.kube/config + --skip-helm-wait install helm charts without waiting for deployments to be ready +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + +## constellation ssh + +Generate a certificate for emergency SSH access + +### Synopsis + +Generate a certificate for emergency SSH access to your SSH-enabled constellation cluster. + +``` +constellation ssh [flags] +``` + +### Options + +``` + -h, --help help for ssh + --key string the path to an existing SSH public key +``` + +### Options inherited from parent commands + +``` + --debug enable debug logging + --force disable version compatibility checks - might result in corrupted clusters + --tf-log string Terraform log level (default "NONE") + -C, --workspace string path to the Constellation workspace +``` + diff --git a/docs/versioned_docs/version-2.23/reference/migration.md b/docs/versioned_docs/version-2.23/reference/migration.md new file mode 100644 index 000000000..eb55d650b --- /dev/null +++ b/docs/versioned_docs/version-2.23/reference/migration.md @@ -0,0 +1,140 @@ +# Migrations + +This document describes breaking changes and migrations between Constellation releases. +Use [`constellation config migrate`](./cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Migrations to v2.23.0 + +### GCP + +GCP will require the additional permission `compute.forwardingRules.list`. Please update your IAM roles using `constellation iam upgrade apply`. + +## Migrations to v2.19.1 + +### Azure + +* During the upgrade, security rules are migrated and the old ones need to be cleaned up manually by the user. The below script shows how to delete them through the Azure CLI: + +```bash +#!/usr/bin/env bash +name="" # the name provided in the config +uid="" # the cluster id can be retrieved via `yq '.infrastructure.uid' constellation-state.yaml` +resource_group="" # the RG can be retrieved via `yq '.provider.azure.resourceGroup' constellation-conf.yaml` + +rules=( + "kubernetes" + "bootstrapper" + "verify" + "recovery" + "join" + "debugd" + "konnectivity" +) + +for rule in "${rules[@]}"; do + echo "Deleting rule: ${rule}" + az network nsg rule delete \ + --resource-group "${resource_group}" \ + --nsg-name "${name}-${uid}" \ + --name "${rule}" +done + +echo "All specified rules have been deleted." +``` + +## Migrating from CLI versions before 2.21.1 + +### AWS + +* AWS clusters that use `LoadBalancer` resources require more IAM permissions. Please upgrade your IAM roles using `constellation iam upgrade apply`. This will show necessary changes and apply them, if desired. + +## Migrating from CLI versions before 2.19.0 + +### Azure + +* To allow seamless upgrades on Azure when Kubernetes services of type `LoadBalancer` are deployed, the target + load balancer in which the `cloud-controller-manager` creates load balancing rules was changed. Instead of using the load balancer + created and maintained by the CLI's Terraform code, the `cloud-controller-manager` now creates its own load balancer in Azure. + If your Constellation has services of type `LoadBalancer`, please remove them before the upgrade and re-apply them + afterward. + +## Migrating from CLI versions before 2.18.0 + +* The `provider.azure.appClientID` and `provider.azure.appClientSecret` fields are no longer supported and should be removed. +* To keep using an existing UAMI, add the `Owner` permission with the scope of your `resourceGroup`. +* Otherwise, simply [create new Constellation IAM credentials](../workflows/config.md#creating-an-iam-configuration) and use the created UAMI. +* To migrate the authentication for an existing cluster on Azure to an UAMI with the necessary permissions: + 1. Remove the `aadClientId` and `aadClientSecret` from the azureconfig secret. + 2. Set `useManagedIdentityExtension` to `true` and use the `userAssignedIdentity` from the Constellation config for the value of `userAssignedIdentityID`. + 3. Restart the CSI driver, cloud controller manager, cluster autoscaler, and Constellation operator pods. + +## Migrating from CLI versions before 2.10 + +* AWS cluster upgrades require additional IAM permissions for the newly introduced `aws-load-balancer-controller`. Please upgrade your IAM roles using `iam upgrade apply`. This will show necessary changes and apply them, if desired. +* The global `nodeGroups` field was added. +* The fields `instanceType`, `stateDiskSizeGB`, and `stateDiskType` for each cloud provider are now part of the configuration of individual node groups. +* The `constellation create` command no longer uses the flags `--control-plane-count` and `--worker-count`. Instead, the initial node count is configured per node group in the `nodeGroups` field. + +## Migrating from CLI versions before 2.9 + +* The `provider.azure.appClientID` and `provider.azure.clientSecretValue` fields were removed to enforce migration to managed identity authentication + +## Migrating from CLI versions before 2.8 + +* The `measurements` field for each cloud service provider was replaced with a global `attestation` field. +* The `confidentialVM`, `idKeyDigest`, and `enforceIdKeyDigest` fields for the Azure cloud service provider were removed in favor of using the global `attestation` field. +* The optional global field `attestationVariant` was replaced by the now required `attestation` field. + +## Migrating from CLI versions before 2.3 + +* The `sshUsers` field was deprecated in v2.2 and has been removed from the configuration in v2.3. + As an alternative for SSH, check the workflow section [Connect to nodes](../workflows/troubleshooting.md#node-shell-access). +* The `image` field for each cloud service provider has been replaced with a global `image` field. Use the following mapping to migrate your configuration: +
+ Show all + + | CSP | old image | new image | + | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | + | AWS | `ami-06b8cbf4837a0a57c` | `v2.2.2` | + | AWS | `ami-02e96dc04a9e438cd` | `v2.2.2` | + | AWS | `ami-028ead928a9034b2f` | `v2.2.2` | + | AWS | `ami-032ac10dd8d8266e3` | `v2.2.1` | + | AWS | `ami-032e0d57cc4395088` | `v2.2.1` | + | AWS | `ami-053c3e49e19b96bdd` | `v2.2.1` | + | AWS | `ami-0e27ebcefc38f648b` | `v2.2.0` | + | AWS | `ami-098cd37f66523b7c3` | `v2.2.0` | + | AWS | `ami-04a87d302e2509aad` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.2` | `v2.2.2` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.1` | `v2.2.1` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.2.0` | `v2.2.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.1.0` | `v2.1.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/2.0.0` | `v2.0.0` | + | Azure | `/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation_CVM/images/constellation/versions/2.0.0` | `v2.0.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-2` | `v2.2.2` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-1` | `v2.2.1` | + | GCP | `projects/constellation-images/global/images/constellation-v2-2-0` | `v2.2.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-1-0` | `v2.1.0` | + | GCP | `projects/constellation-images/global/images/constellation-v2-0-0` | `v2.0.0` | + +
+* The `enforcedMeasurements` field has been removed and merged with the `measurements` field. + * To migrate your config containing a new image (`v2.3` or greater), remove the old `measurements` and `enforcedMeasurements` entries from your config and run `constellation fetch-measurements` + * To migrate your config containing an image older than `v2.3`, remove the `enforcedMeasurements` entry and replace the entries in `measurements` as shown in the example below: + + ```diff + measurements: + - 0: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + 0: + + expected: DzXCFGCNk8em5ornNZtKi+Wg6Z7qkQfs5CfE3qTkOc8= + + warnOnly: true + - 8: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + 8: + + expected: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + + warnOnly: false + -enforcedMeasurements: + - - 8 + ``` diff --git a/docs/versioned_docs/version-2.23/reference/slsa.md b/docs/versioned_docs/version-2.23/reference/slsa.md new file mode 100644 index 000000000..21f4e713c --- /dev/null +++ b/docs/versioned_docs/version-2.23/reference/slsa.md @@ -0,0 +1,73 @@ +# Supply chain levels for software artifacts (SLSA) adoption + +[Supply chain Levels for Software Artifacts, or SLSA (salsa)](https://slsa.dev/) is a framework for improving and grading a project's build system and engineering processes. SLSA focuses on security improvements for source code storage as well as build system definition, execution, and observation. SLSA is structured in [four levels](https://slsa.dev/spec/v0.1/levels). This page describes the adoption of SLSA for Constellation. + +:::info +SLSA is still in alpha status. The presented levels and their requirements might change in the future. We will adopt any changes into our engineering processes, as they get defined. +::: + +## Level 1 - Adopted + +**[Build - Scripted](https://slsa.dev/spec/v0.1/requirements#scripted-build)** + +All build steps are automated via [Bazel](https://github.com/edgelesssys/constellation/tree/main/bazel/ci) and [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Available](https://slsa.dev/spec/v0.1/requirements#available)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). + +## Level 2 - Adopted + +**[Source - Version Controlled](https://slsa.dev/spec/v0.1/requirements#version-controlled)** + +Constellation is hosted on GitHub using git. + +**[Build - Build Service](https://slsa.dev/spec/v0.1/requirements#build-service)** + +All builds are carried out by [GitHub Actions](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Provenance - Authenticated](https://slsa.dev/spec/v0.1/requirements#authenticated)** + +Provenance for the CLI is signed using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator). Learn [how to verify the CLI](../workflows/verify-cli.md) using the signed provenance, before using it for the first time. + +**[Provenance - Service Generated](https://slsa.dev/spec/v0.1/requirements#service-generated)** + +Provenance for the CLI is generated using the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) in GitHub Actions. + +## Level 3 - Adopted + +**[Source - Verified History](https://slsa.dev/spec/v0.1/requirements#verified-history)** + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization [requires two-factor authentication](https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-two-factor-authentication-for-your-organization/requiring-two-factor-authentication-in-your-organization) for all members. + +**[Source - Retained Indefinitely](https://slsa.dev/spec/v0.1/requirements#retained-indefinitely)** + +Since we use GitHub to host the repository, an external person can't modify or delete the history. Before a pull request can be merged, an explicit approval from an [Edgeless Systems](https://github.com/edgelesssys) team member is required. + +The same holds true for changes proposed by team members. Each change to `main` needs to be proposed via a pull request and requires at least one approval. + +The [Edgeless Systems](https://github.com/edgelesssys) GitHub organization admins control these settings and are able to make changes to the repository's history should legal requirements necessitate it. These changes require two-party approval following the obliterate policy. + +**[Build - Build as Code](https://slsa.dev/spec/v0.1/requirements#build-as-code)** + +All build files for Constellation are stored in [the same repository](https://github.com/edgelesssys/constellation/tree/main/.github). + +**[Build - Ephemeral Environment](https://slsa.dev/spec/v0.1/requirements#ephemeral-environment)** + +All GitHub Action workflows are executed on [GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). These runners are only available during workflow. + +We currently don't use [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). + +**[Build - Isolated](https://slsa.dev/spec/v0.1/requirements#isolated)** + +As outlined in the previous section, we use GitHub-hosted runners, which provide a new, isolated and ephemeral environment for each build. + +Additionally, the [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator#generation-of-provenance) itself is run in an isolated workflow with the artifact hash as defined inputs. + +**[Provenance - Non-falsifiable](https://slsa.dev/spec/v0.1/requirements#non-falsifiable)** + +As outlined by [SLSA GitHub generator](https://github.com/slsa-framework/slsa-github-generator) it already fulfills the non-falsifiable requirements for SLSA Level 3. The generated provenance is signed using [sigstore](https://sigstore.dev/) with an OIDC based proof of identity. + +## Level 4 - In Progress + +We strive to adopt certain aspect of SLSA Level 4 that support our engineering process. At the same time, SLSA is still in alpha status and the biggest changes to SLSA are expected to be around Level 4. diff --git a/docs/versioned_docs/version-2.23/reference/terraform.md b/docs/versioned_docs/version-2.23/reference/terraform.md new file mode 100644 index 000000000..9825a8bb8 --- /dev/null +++ b/docs/versioned_docs/version-2.23/reference/terraform.md @@ -0,0 +1,37 @@ +# Terraform usage + +[Terraform](https://www.terraform.io/) is an Infrastructure as Code (IaC) framework to manage cloud resources. This page explains how Constellation uses it internally and how advanced users may manually use it to have more control over the resource creation. + +:::info +Information on this page is intended for users who are familiar with Terraform. +It's not required for common usage of Constellation. +See the [Terraform documentation](https://developer.hashicorp.com/terraform/docs) if you want to learn more about it. +::: + +## Terraform state files + +Constellation keeps Terraform state files in subdirectories of the workspace together with the corresponding Terraform configuration files and metadata. +The subdirectories are created on the first Constellation CLI action that uses Terraform internally. + +Currently, these subdirectories are: + +* `constellation-terraform` - Terraform state files for the resources of the Constellation cluster +* `constellation-iam-terraform` - Terraform state files for IAM configuration + +As with all commands, commands that work with these files (e.g., `apply`, `terminate`, `iam`) have to be executed from the root of the cluster's [workspace directory](../architecture/orchestration.md#workspaces). You usually don't need and shouldn't manipulate or delete the subdirectories manually. + +## Interacting with Terraform manually + +Manual interaction with Terraform state created by Constellation (i.e., via the Terraform CLI) should only be performed by experienced users. It may lead to unrecoverable loss of cloud resources. For the majority of users and use cases, the interaction done by the [Constellation CLI](cli.md) is sufficient. + +## Terraform debugging + +To debug Terraform issues, the Constellation CLI offers the `tf-log` flag. You can set it to any of [Terraform's log levels](https://developer.hashicorp.com/terraform/internals/debugging): +* `JSON` (JSON-formatted logs at `TRACE` level) +* `TRACE` +* `DEBUG` +* `INFO` +* `WARN` +* `ERROR` + +The log output is written to the `terraform.log` file in the workspace directory. The output is appended to the file on each run. diff --git a/docs/versioned_docs/version-2.23/workflows/cert-manager.md b/docs/versioned_docs/version-2.23/workflows/cert-manager.md new file mode 100644 index 000000000..1d847e8bf --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/cert-manager.md @@ -0,0 +1,13 @@ +# Install cert-manager + +:::caution +If you want to use cert-manager with Constellation, pay attention to the following to avoid potential pitfalls. +::: + +Constellation ships with cert-manager preinstalled. +The default installation is part of the `kube-system` namespace, as all other Constellation-managed microservices. +You are free to install more instances of cert-manager into other namespaces. +However, be aware that any new installation needs to use the same version as the one installed with Constellation or rely on the same CRD versions. +Also remember to set the `installCRDs` value to `false` when installing new cert-manager instances. +It will create problems if you have two installations of cert-manager depending on different versions of the installed CRDs. +CRDs are cluster-wide resources and cert-manager depends on specific versions of those CRDs for each release. diff --git a/docs/versioned_docs/version-2.23/workflows/config.md b/docs/versioned_docs/version-2.23/workflows/config.md new file mode 100644 index 000000000..7868ff1be --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/config.md @@ -0,0 +1,353 @@ +# Configure your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Before you can create your cluster, you need to configure the identity and access management (IAM) for your cloud service provider (CSP) and choose machine types for the nodes. + +## Creating the configuration file + +You can generate a configuration file for your CSP by using the following CLI command: + + + + +```bash +constellation config generate aws +``` + + + + +```bash +constellation config generate azure +``` + + + + +```bash +constellation config generate gcp +``` + + + + +```bash +constellation config generate stackit +``` + + + + +This creates the file `constellation-conf.yaml` in the current directory. + +## Choosing a VM type + +Constellation supports the following VM types: + + + +By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. +If you are using the default attestation variant `awsSEVSNP`, you can use the instance types described in [AWS's AMD SEV-SNP docs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html). +Please mind the region restrictions mentioned in the [Getting started](../getting-started/first-steps.md#create-a-cluster) section. + +If you are using the attestation variant `awsNitroTPM`, you can choose any of the [nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. + +You can also run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. + + + + +By default, Constellation uses `m1a.4cd` VMs (4 vCPUs, 30 GB RAM) to create your cluster. +Optionally, you can switch to a different VM type by modifying `instanceType` in the configuration file. + +The following instance types are known to be supported: + +| name | vCPUs | GB RAM | +|----------|-------|--------| +| m1a.4cd | 4 | 30 | +| m1a.8cd | 8 | 60 | +| m1a.16cd | 16 | 120 | +| m1a.30cd | 30 | 230 | + +You can choose any of the SEV-enabled instance types. You can find a list of all supported instance types in the [STACKIT documentation](https://docs.stackit.cloud/stackit/en/virtual-machine-flavors-75137231.html). + +The Constellation CLI can also print the supported instance types with: `constellation config instance-types`. + + + + +Fill the desired VM type into the `instanceType` fields in the `constellation-conf.yml` file. + +## Creating additional node groups + +By default, Constellation creates the node groups `control_plane_default` and `worker_default` for control-plane nodes and workers, respectively. +If you require additional control-plane or worker groups with different instance types, zone placements, or disk sizes, you can add additional node groups to the `constellation-conf.yml` file. +Each node group can be scaled individually. + +Consider the following example for AWS: + +```yaml +nodeGroups: + control_plane_default: + role: control-plane + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 3 + worker_default: + role: worker + instanceType: c6a.xlarge + stateDiskSizeGB: 30 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 2 + high_cpu: + role: worker + instanceType: c6a.24xlarge + stateDiskSizeGB: 128 + stateDiskType: gp3 + zone: eu-west-1c + initialCount: 1 +``` + +This configuration creates an additional node group `high_cpu` with a larger instance type and disk. + +You can use the field `zone` to specify what availability zone nodes of the group are placed in. +On Azure, this field is empty by default and nodes are automatically spread across availability zones. +STACKIT currently offers SEV-enabled CPUs in the `eu01-1`, `eu01-2`, and `eu01-3` zones. +Consult the documentation of your cloud provider for more information: + +* [AWS](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) +* [Azure](https://azure.microsoft.com/en-us/explore/global-infrastructure/availability-zones) +* [GCP](https://cloud.google.com/compute/docs/regions-zones) +* [STACKIT](https://docs.stackit.cloud/stackit/en/regions-and-availability-zones-75137212.html) + +## Choosing a Kubernetes version + +To learn which Kubernetes versions can be installed with your current CLI, you can run `constellation config kubernetes-versions`. +See also Constellation's [Kubernetes support policy](../architecture/versions.md#kubernetes-support-policy). + +## Creating an IAM configuration + +You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. +If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. + + + + +You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create aws --zone=us-east-2a --prefix=constellTest +``` + +This command creates IAM configuration for the AWS zone `us-east-2a` using the prefix `constellTest` for all named resources being created. + +Constellation OS images are currently replicated to the following regions: + +* `eu-central-1` +* `eu-west-1` +* `eu-west-3` +* `us-east-2` +* `ap-south-1` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + +You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create azure --subscriptionID 00000000-0000-0000-0000-000000000000 --region=westus --resourceGroup=constellTest --servicePrincipal=spTest +``` + +This command creates IAM configuration on the Azure region `westus` creating a new resource group `constellTest` and a new service principal `spTest`. + +CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + +* `germanywestcentral` +* `westus` +* `eastus` +* `northeurope` +* `westeurope` +* `southeastasia` + +If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + +You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). + +```bash +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --prefix=constell-test +``` + +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. + +Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. + +Paste the output into the corresponding fields of the `constellation-conf.yaml` file. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + + +
+Alternatively, you can manually create the IAM configuration on your CSP. + +The following describes the configuration fields and how you obtain the required information or create the required resources. + + + + +* **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. + + Constellation OS images are currently replicated to the following regions: + * `eu-central-1` + * `eu-west-1` + * `eu-west-3` + * `us-east-2` + * `ap-south-1` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+AWS+image+region:+xx-xxxx-x). + + You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). + +* **zone**: The name of your chosen AWS data center availability zone, e.g., `us-east-2a`. + + Learn more about [availability zones in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones). + +* **iamProfileControlPlane**: The name of an IAM instance profile attached to all control-plane nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `control_plane_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.control_plane_policy`. + +* **iamProfileWorkerNodes**: The name of an IAM instance profile attached to all worker nodes. + + You can create the resource with [Terraform](https://www.terraform.io/). For that, use the [provided Terraform script](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam) to generate the necessary profile. The profile name will be provided as Terraform output value: `worker_nodes_instance_profile_name`. + + Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. + + + + +* **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. + + You can view your subscription UUID via `az account show` and read the `id` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription). + +* **tenant**: The UUID of your Azure tenant, e.g., `3400e5a2-8fe2-492a-886c-38cb66170f25`. + + You can view your tenant UUID via `az account show` and read the `tenant` field. For more information refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-ad-tenant). + +* **location**: The Azure datacenter location you want to deploy your cluster in, e.g., `westus`. + + CVMs are available in several Azure regions. Constellation OS images are currently replicated to the following: + + * `germanywestcentral` + * `westus` + * `eastus` + * `northeurope` + * `westeurope` + * `southeastasia` + + If you require the OS image to be available in another region, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md&title=Support+new+Azure+image+region:+xx-xxxx-x). + + You can find a list of all [regions in Azure's documentation](https://azure.microsoft.com/en-us/global-infrastructure/services/?products=virtual-machines®ions=all). + +* **resourceGroup**: [Create a new resource group in Azure](https://learn.microsoft.com/azure/azure-resource-manager/management/manage-resource-groups-portal) for your Constellation cluster. Set this configuration field to the name of the created resource group. + +* **userAssignedIdentity**: [Create a new managed identity in Azure](https://learn.microsoft.com/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). You should create the identity in a different resource group as all resources within the cluster resource group will be deleted on cluster termination. + + Add three role assignments to the identity: `Owner`, `Virtual Machine Contributor`, and `Application Insights Component Contributor`. The `scope` of all three should refer to the previously created cluster resource group. + + Set the configuration value to the full ID of the created identity, e.g., `/subscriptions/8b8bd01f-efd9-4113-9bd1-c82137c32da7/resourcegroups/constellation-identity/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-identity`. You can get it by opening the `JSON View` from the `Overview` section of the identity. + + The user-assigned identity is used by instances of the cluster to access other cloud resources. + For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). + + + + +* **project**: The ID of your GCP project, e.g., `constellation-129857`. + + You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). + +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. + + You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. + + You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). + +* **serviceAccountKeyPath**: To configure this, you need to create a GCP [service account](https://cloud.google.com/iam/docs/service-accounts) with the following permissions: + + * `Compute Instance Admin (v1) (roles/compute.instanceAdmin.v1)` + * `Compute Network Admin (roles/compute.networkAdmin)` + * `Compute Security Admin (roles/compute.securityAdmin)` + * `Compute Storage Admin (roles/compute.storageAdmin)` + * `Service Account User (roles/iam.serviceAccountUser)` + + Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. + + + + +STACKIT requires manual creation and configuration of service accounts. Look at the [first steps](../getting-started/first-steps.md) for more information. + + + +
+ +Now that you've configured your CSP, you can [create your cluster](./create.md). + +## Deleting an IAM configuration + +You can keep a created IAM configuration and reuse it for new clusters. Alternatively, you can also delete it if you don't want to use it anymore. + +Delete the IAM configuration by executing the following command in the same directory where you executed `constellation iam create` (the directory that contains [`constellation-iam-terraform`](../reference/terraform.md) as a subdirectory): + +```bash +constellation iam destroy +``` + +:::caution +For Azure, deleting the IAM configuration by executing `constellation iam destroy` will delete the whole resource group created by `constellation iam create`. +This also includes any additional resources in the resource group that weren't created by Constellation. +::: diff --git a/docs/versioned_docs/version-2.23/workflows/create.md b/docs/versioned_docs/version-2.23/workflows/create.md new file mode 100644 index 000000000..6074ebb16 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/create.md @@ -0,0 +1,93 @@ +# Create your cluster + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Creating your cluster happens through multiple phases. +The most significant ones are: + +1. Creating the necessary resources in your cloud environment +2. Bootstrapping the Constellation cluster and setting up a connection +3. Installing the necessary Kubernetes components + +`constellation apply` handles all this in a single command. +You can use the `--skip-phases` flag to skip specific phases of the process. +For example, if you created the infrastructure manually, you can skip the cloud resource creation phase. + +See the [architecture](../architecture/orchestration.md) section for details on the inner workings of this process. + +:::tip +If you don't have a cloud subscription, you can also set up a [local Constellation cluster using virtualization](../getting-started/first-steps-local.md) for testing. +::: + +Before you create the cluster, make sure to have a [valid configuration file](./config.md). + + + + +```bash +constellation apply +``` + +`apply` stores the state of your cluster's cloud resources in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. + + + + +Self-managed infrastructure allows for more flexibility in the setup, by separating the infrastructure setup from the Constellation cluster management. +This provides flexibility in DevOps and can meet potential regulatory requirements. +It's recommended to use Terraform for infrastructure management, but you can use any tool of your choice. + +:::info + + When using Terraform, you can use the [Constellation Terraform provider](./terraform-provider.md) to manage the entire Constellation cluster lifecycle. + +::: + +You can refer to the Terraform files for the selected CSP from the [Constellation GitHub repository](https://github.com/edgelesssys/constellation/tree/main/terraform/infrastructure) for a minimum Constellation cluster configuration. From this base, you can now add, edit, or substitute resources per your own requirements with the infrastructure +management tooling of your choice. You need to keep the essential functionality of the base configuration in order for your cluster to function correctly. + + + +:::info + + On Azure, a manual update to the MAA provider's policy is necessary. + You can apply the update with the following command after creating the infrastructure, with `` being the URL of the MAA provider (i.e., `$(terraform output attestation_url | jq -r)`, when using the minimal Terraform configuration). + + ```bash + constellation maa-patch + ``` + +::: + + + +Make sure all necessary resources are created, e.g., through checking your CSP's portal and retrieve the necessary values, aligned with the outputs (specified in `outputs.tf`) of the base configuration. + +Fill these outputs into the corresponding fields of the `Infrastructure` block inside the `constellation-state.yaml` file. For example, fill the IP or DNS name your cluster can be reached at into the `.Infrastructure.ClusterEndpoint` field. + +With the required cloud resources set up, continue with initializing your cluster. + +```bash +constellation apply --skip-phases=infrastructure +``` + + + + +Finally, configure `kubectl` for your cluster: + +```bash +export KUBECONFIG="$PWD/constellation-admin.conf" +``` + +🏁 That's it. You've successfully created a Constellation cluster. + +### Troubleshooting + +In case `apply` fails, the CLI collects logs from the bootstrapping instance and stores them inside `constellation-cluster.log`. diff --git a/docs/versioned_docs/version-2.23/workflows/lb.md b/docs/versioned_docs/version-2.23/workflows/lb.md new file mode 100644 index 000000000..868e61076 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/lb.md @@ -0,0 +1,28 @@ +# Expose a service + +Constellation integrates the native load balancers of each CSP. Therefore, to expose a service simply [create a service of type `LoadBalancer`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Internet-facing LB service on AWS + +To expose your application service externally you might want to use a Kubernetes Service of type `LoadBalancer`. On AWS, load-balancing is achieved through the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller) as in the managed EKS. + +Since recent versions, the controller deploy an internal LB by default requiring to set an annotation `service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing` to have an internet-facing LB. For more details, see the [official docs](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/service/nlb/). + +For general information on LB with AWS see [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html). + +:::caution +Before terminating the cluster, all LB backed services should be deleted, so that the controller can cleanup the related resources. +::: + +## Ingress on AWS + +The AWS Load Balancer Controller also provisions `Ingress` resources of class `alb`. +AWS Application Load Balancers (ALBs) can be configured with a [`target-type`](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.7/guide/ingress/annotations/#target-type). +The target type `ip` requires using the EKS container network solution, which makes it incompatible with Constellation. +If a service can be exposed on a `NodePort`, the target type `instance` can be used. + +See [Application load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html) for more information. + +:::caution +Ingress handlers backed by AWS ALBs reside outside the Constellation cluster, so they shouldn't be handling sensitive traffic! +::: diff --git a/docs/versioned_docs/version-2.23/workflows/recovery.md b/docs/versioned_docs/version-2.23/workflows/recovery.md new file mode 100644 index 000000000..592ae247b --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/recovery.md @@ -0,0 +1,179 @@ +# Recover your cluster + +Recovery of a Constellation cluster means getting it back into a healthy state after too many concurrent node failures in the control plane. +Reasons for an unhealthy cluster can vary from a power outage, or planned reboot, to migration of nodes and regions. +Recovery events are rare, because Constellation is built for high availability and automatically and securely replaces failed nodes. When a node is replaced, Constellation's control plane first verifies the new node before it sends the node the cryptographic keys required to decrypt its [state disk](../architecture/images.md#state-disk). + +Constellation provides a recovery mechanism for cases where the control plane has failed and is unable to replace nodes. +The `constellation recover` command securely connects to all nodes in need of recovery using [attested TLS](../architecture/attestation.md#attested-tls-atls) and provides them with the keys to decrypt their state disks and continue booting. + +## Identify unhealthy clusters + +The first step to recovery is identifying when a cluster becomes unhealthy. +Usually, this can be first observed when the Kubernetes API server becomes unresponsive. + +You can check the health status of the nodes via the cloud service provider (CSP). +Constellation provides logging information on the boot process and status via serial console output. +In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. + + + + +First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. In the ASG's *Instance management* view, select each desired instance. In the upper right corner, select **Action > Monitor and troubleshoot > Get system log**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +In the Azure portal, find the cluster's resource group. +Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. +On the left, go to **Settings** > **Instances** and check that enough members are in a *Running* state. + +Second, check the boot logs of these *Instances*. +In the scale set's *Instances* view, open the details page of the desired instance. +On the left, go to **Support + troubleshooting** > **Serial console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T09:56:41Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"azure"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["10.9.0.5:30090","10.9.0.6:30090"]} +{"level":"INFO","ts":"2022-09-08T09:56:43Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.5:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.5:30090: i/o timeout\"","endpoint":"10.9.0.5:30090"} +{"level":"INFO","ts":"2022-09-08T09:57:03Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"10.9.0.6:30090"} +{"level":"WARN","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 10.9.0.6:30090: i/o timeout\"","endpoint":"10.9.0.6:30090"} +{"level":"ERROR","ts":"2022-09-08T09:57:23Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, check that the control plane *Instance Group* has enough members in a *Ready* state. +In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. + +Second, check the status of the *VM Instances*. +Go to **VM Instances** and open the details of the desired instance. +Check the serial console output of that instance by opening the **Logs** > **Serial port 1 (console)** page: + +![GCP portal serial console link](../_media/recovery-gcp-serial-console-link.png) + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +First, open the STACKIT portal to view all servers in your project. Select individual control plane nodes `--control-plane--` and check that enough members are in a *Running* state. + +Second, check the boot logs of these servers. Click on a server name and select **Overview**. Find the **Machine Setup** section and click on **Web console** > **Open console**. + +In the serial console output, search for `Waiting for decryption key`. +Similar output to the following means your node was restarted and needs to decrypt the [state disk](../architecture/images.md#state-disk): + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","caller":"cmd/main.go:55","msg":"Starting disk-mapper","version":"2.0.0","cloudProvider":"gcp"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"setupManager","caller":"setup/setup.go:72","msg":"Preparing existing state disk"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:65","msg":"Starting RejoinClient"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"recoveryServer","caller":"recoveryserver/server.go:59","msg":"Starting RecoveryServer"} +``` + +The node will then try to connect to the [*JoinService*](../architecture/microservices.md#joinservice) and obtain the decryption key. +If this fails due to an unhealthy control plane, you will see log messages similar to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:77","msg":"Received list with JoinService endpoints","endpoints":["192.168.178.4:30090","192.168.178.2:30090"]} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.4:30090"} +{"level":"WARN","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.4:30090: connect: connection refused\"","endpoint":"192.168.178.4:30090"} +{"level":"INFO","ts":"2022-09-08T10:21:53Z","logger":"rejoinClient","caller":"rejoinclient/client.go:96","msg":"Requesting rejoin ticket","endpoint":"192.168.178.2:30090"} +{"level":"WARN","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:101","msg":"Failed to rejoin on endpoint","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.178.2:30090: i/o timeout\"","endpoint":"192.168.178.2:30090"} +{"level":"ERROR","ts":"2022-09-08T10:22:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:110","msg":"Failed to rejoin on all endpoints"} +``` + +This means that you have to recover the node manually. + + + + +## Recover a cluster + +Recovering a cluster requires the following parameters: + +* The `constellation-state.yaml` file in your working directory or the cluster's endpoint +* The master secret of the cluster + +A cluster can be recovered like this: + +```bash +$ constellation recover +Pushed recovery key. +Pushed recovery key. +Pushed recovery key. +Recovered 3 control-plane nodes. +``` + +In the serial console output of the node you'll see a similar output to the following: + +```json +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:93","msg":"Received recover call"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer","caller":"recoveryserver/server.go:125","msg":"Received state disk key and measurement secret, shutting down server"} +{"level":"INFO","ts":"2022-09-08T10:26:59Z","logger":"recoveryServer.gRPC","caller":"zap/server_interceptors.go:61","msg":"finished streaming call with code OK","grpc.start_time":"2022-09-08T10:26:59Z","system":"grpc","span.kind":"server","grpc.service":"recoverproto.API","grpc.method":"Recover","peer.address":"192.0.2.3:41752","grpc.code":"OK","grpc.time_ms":15.701} +{"level":"INFO","ts":"2022-09-08T10:27:13Z","logger":"rejoinClient","caller":"rejoinclient/client.go:87","msg":"RejoinClient stopped"} +``` diff --git a/docs/versioned_docs/version-2.23/workflows/reproducible-builds.md b/docs/versioned_docs/version-2.23/workflows/reproducible-builds.md new file mode 100644 index 000000000..e3bc46095 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/reproducible-builds.md @@ -0,0 +1,63 @@ +# Reproduce released artifacts + +Constellation has first-class support for [reproducible builds](https://reproducible-builds.org). +Reproducing the released artifacts is an alternative to [signature verification](verify-cli.md) that doesn't require trusting Edgeless Systems' release process. +The following sections describe how to rebuild an artifact and how Constellation ensures that the rebuild reproduces the artifacts bit-by-bit. + +## Build environment prerequisites + +The build systems used by Constellation - [Bazel](https://bazel.build/) and [Nix](https://nixos.org) - are designed for deterministic, reproducible builds. +These two dependencies should be the only prerequisites for a successful build. +However, it can't be ruled out completely that peculiarities of the host affect the build result. +Thus, we recommend the following host setup for best results: + +1. A Linux operating system not older than v5.4. +2. The GNU C library not older than v2.31 (avoid `musl`). +3. GNU `coreutils` not older than v8.30 (avoid `busybox`). +4. An `ext4` filesystem for building. +5. AppArmor turned off. + +This is given, for example, on an Ubuntu 22.04 system, which is also used for reproducibility tests. + +:::note + +To avoid any backwards-compatibility issues, the host software versions should also not be much newer than the Constellation release. + +::: + +## Run the build + +The following instructions outline qualitatively how to reproduce a build. +Constellation implements these instructions in the [Reproducible Builds workflow](https://github.com/edgelesssys/constellation/actions/workflows/reproducible-builds.yml), which continuously tests for reproducibility. +The workflow is a good place to look up specific version numbers and build steps. + +1. Check out the Constellation repository at the tag corresponding to the release. + + ```bash + git clone https://github.com/edgelesssys/constellation.git + cd constellation + git checkout v2.20.0 + ``` + +2. [Install the Bazel release](https://bazel.build/install) specified in `.bazelversion`. +3. [Install Nix](https://nixos.org/download/) (any recent version should do). +4. Run the build with `bazel build $target` for one of the following targets of interest: + + ```data + //cli:cli_enterprise_darwin_amd64 + //cli:cli_enterprise_darwin_arm64 + //cli:cli_enterprise_linux_amd64 + //cli:cli_enterprise_linux_arm64 + //cli:cli_enterprise_windows_amd64 + ``` + +5. Compare the build result with the downloaded release artifact. + + + +## Feedback + +Reproduction failures often indicate a bug in the build system or in the build definitions. +Therefore, we're interested in any reproducibility issues you might encounter. +[Start a bug report](https://github.com/edgelesssys/constellation/issues/new/choose) and describe the details of your build environment. +Make sure to include your result binary or a [`diffoscope`](https://diffoscope.org/) report, if possible. diff --git a/docs/versioned_docs/version-2.23/workflows/s3proxy.md b/docs/versioned_docs/version-2.23/workflows/s3proxy.md new file mode 100644 index 000000000..121e8a461 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/s3proxy.md @@ -0,0 +1,58 @@ +# Install s3proxy + +Constellation includes a transparent client-side encryption proxy for [AWS S3](https://aws.amazon.com/de/s3/) and compatible stores. +s3proxy encrypts objects before sending them to S3 and automatically decrypts them on retrieval, without requiring changes to your application. +With s3proxy, you can use S3 for storage in a confidential way without having to trust the storage provider. + +## Limitations + +Currently, s3proxy has the following limitations: +- Only `PutObject` and `GetObject` requests are encrypted/decrypted by s3proxy. +By default, s3proxy will block requests that may expose unencrypted data to S3 (e.g. UploadPart). +The `allow-multipart` flag disables request blocking for evaluation purposes. +- Using the [Range](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html#API_GetObject_RequestSyntax) header on `GetObject` is currently not supported and will result in an error. + +These limitations will be removed with future iterations of s3proxy. +If you want to use s3proxy but these limitations stop you from doing so, consider [opening an issue](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&projects=&template=feature_request.yml). + +## Deployment + +You can add the s3proxy to your Constellation cluster as follows: +1. Add the Edgeless Systems chart repository: + ```bash + helm repo add edgeless https://helm.edgeless.systems/stable + helm repo update + ``` +2. Set ACCESS_KEY and ACCESS_SECRET to valid credentials you want s3proxy to use to interact with S3. +3. Deploy s3proxy: + ```bash + helm install s3proxy edgeless/s3proxy --set awsAccessKeyID="$ACCESS_KEY" --set awsSecretAccessKey="$ACCESS_SECRET" + ``` + +If you want to run a demo application, check out the [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example. + + +## Technical details + +### Encryption + +s3proxy relies on Google's [Tink Cryptographic Library](https://developers.google.com/tink) to implement cryptographic operations securely. +The used cryptographic primitives are [NIST SP 800 38f](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf) for key wrapping and [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)-[GCM](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Galois/counter_(GCM)) with 256 bit keys for data encryption. + +s3proxy uses [envelope encryption](https://cloud.google.com/kms/docs/envelope-encryption) to encrypt objects. +This means s3proxy uses a key encryption key (KEK) issued by the [KeyService](../architecture/microservices.md#keyservice) to encrypt data encryption keys (DEKs). +Each S3 object is encrypted with its own DEK. +The encrypted DEK is then saved as metadata of the encrypted object. +This enables key rotation of the KEK without re-encrypting the data in S3. +The approach also allows access to objects from different locations, as long as each location has access to the KEK. + +### Traffic interception + +To use s3proxy, you have to redirect your outbound S3 traffic to s3proxy. +This can either be done by modifying your client application or by changing the deployment of your application. + +The necessary deployment modifications are to add DNS redirection and a trusted TLS certificate to the client's trust store. +DNS redirection can be defined for each pod, allowing you to use s3proxy for one application without changing other applications in the same cluster. +Adding a trusted TLS certificate is necessary as clients communicate with s3proxy via HTTPS. +To have your client application trust s3proxy's TLS certificate, the certificate has to be added to the client's certificate trust store. +The [Filestash with s3proxy](../getting-started/examples/filestash-s3proxy.md) example shows how to do this. diff --git a/docs/versioned_docs/version-2.23/workflows/sbom.md b/docs/versioned_docs/version-2.23/workflows/sbom.md new file mode 100644 index 000000000..6c1702dee --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/sbom.md @@ -0,0 +1,93 @@ +# Consume software bill of materials (SBOMs) + + + +--- + +Constellation builds produce a [software bill of materials (SBOM)](https://www.ntia.gov/SBOM) for each generated [artifact](../architecture/microservices.md). +You can use SBOMs to make informed decisions about dependencies and vulnerabilities in a given application. Enterprises rely on SBOMs to maintain an inventory of used applications, which allows them to take data-driven approaches to managing risks related to vulnerabilities. + +SBOMs for Constellation are generated using [Syft](https://github.com/anchore/syft), signed using [Cosign](https://github.com/sigstore/cosign), and stored with the produced artifact. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). + +Make sure the key is available in a file named `cosign.pub` to execute the following examples. +::: + +## Verify and download SBOMs + +The following sections detail how to work with each type of artifact to verify and extract the SBOM. + +### Constellation CLI + +The SBOM for Constellation CLI is made available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). The SBOM (`constellation.spdx.sbom`) and corresponding signature (`constellation.spdx.sbom.sig`) are valid for each Constellation CLI for a given version, regardless of architecture and operating system. + +```bash +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom +curl -LO https://github.com/edgelesssys/constellation/releases/download/v2.2.0/constellation.spdx.sbom.sig +cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig constellation.spdx.sbom +``` + +### Container Images + +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. + +As a consumer, use cosign to download and verify the SBOM: + +```bash +# Verify and download the attestation statement +cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.att.json +# Extract SBOM from attestation statement +jq -r .payload verification-service.att.json | base64 -d > verification-service.cyclonedx.sbom +``` + +A successful verification should result in similar output: + +```shell-session +$ cosign verify-attestation ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 --type 'https://cyclonedx.org/bom' --key cosign.pub --output-file verification-service.sbom + +Verification for ghcr.io/edgelesssys/constellation/verification-service@v2.2.0 -- +The following checks were performed on each of these signatures: + - The cosign claims were validated + - The signatures were verified against the specified public key +$ jq -r .payload verification-service.sbom | base64 -d > verification-service.cyclonedx.sbom +``` + +:::note + +This example considers only the `verification-service`. The same approach works for all containers in the [Constellation container registry](https://github.com/orgs/edgelesssys/packages?repo_name=constellation). + +::: + + + +## Vulnerability scanning + +You can use a plethora of tools to consume SBOMs. This section provides suggestions for tools that are popular and known to produce reliable results, but any tool that consumes [SPDX](https://spdx.dev/) or [CycloneDX](https://cyclonedx.org/) files should work. + +Syft is able to [convert between the two formats](https://github.com/anchore/syft#format-conversion-experimental) in case you require a specific type. + +### Grype + +[Grype](https://github.com/anchore/grype) is a CLI tool that lends itself well for integration into CI/CD systems or local developer machines. It's also able to consume the signed attestation statement directly and does the verification in one go. + +```bash +grype att:verification-service.sbom --key cosign.pub --add-cpes-if-none -q +``` + +### Dependency Track + +[Dependency Track](https://dependencytrack.org/) is one of the oldest and most mature solutions when it comes to managing software inventory and vulnerabilities. Once imported, it continuously scans SBOMs for new vulnerabilities. It supports the CycloneDX format and provides direct guidance on how to comply with [U.S. Executive Order 14028](https://docs.dependencytrack.org/usage/executive-order-14028/). diff --git a/docs/versioned_docs/version-2.23/workflows/scale.md b/docs/versioned_docs/version-2.23/workflows/scale.md new file mode 100644 index 000000000..28f19e3f1 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/scale.md @@ -0,0 +1,122 @@ +# Scale your cluster + +Constellation provides all features of a Kubernetes cluster including scaling and autoscaling. + +## Worker node scaling + +### Autoscaling + +Constellation comes with autoscaling disabled by default. To enable autoscaling, find the scaling group of +worker nodes: + +```bash +kubectl get scalinggroups -o json | yq '.items | .[] | select(.spec.role == "Worker") | [{"name": .metadata.name, "nodeGoupName": .spec.nodeGroupName}]' +``` + +This will output a list of scaling groups with the corresponding cloud provider name (`name`) and the cloud provider agnostic name of the node group (`nodeGroupName`). + +Then, patch the `autoscaling` field of the scaling group resource with the desired `name` to `true`: + +```bash +# Replace with the name of the scaling group you want to enable autoscaling for +worker_group= +kubectl patch scalinggroups $worker_group --patch '{"spec":{"autoscaling": true}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler now automatically provisions additional worker nodes so that all pods have a place to run. +You can configure the minimum and maximum number of worker nodes in the scaling group by patching the `min` or +`max` fields of the scaling group resource: + +```bash +kubectl patch scalinggroups $worker_group --patch '{"spec":{"max": 5}}' --type='merge' +kubectl get scalinggroup $worker_group -o jsonpath='{.spec}' | yq -P +``` + +The cluster autoscaler will now never provision more than 5 worker nodes. + +If you want to see the autoscaling in action, try to add a deployment with a lot of replicas, like the +following Nginx deployment. The number of replicas needed to trigger the autoscaling depends on the size of +and count of your worker nodes. Wait for the rollout of the deployment to finish and compare the number of +worker nodes before and after the deployment: + +```bash +kubectl create deployment nginx --image=nginx --replicas 150 +kubectl -n kube-system get nodes +kubectl rollout status deployment nginx +kubectl -n kube-system get nodes +``` + +### Manual scaling + +Alternatively, you can manually scale your cluster up or down: + + + + +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-workers`. +3. Go to **settings** and **scaling**. +4. Set the new **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **worker** instance group. +3. Set the new **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +## Control-plane node scaling + +Control-plane nodes can **only be scaled manually and only scaled up**! + +To increase the number of control-plane nodes, follow these steps: + + + + +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. + + + + +1. Find your Constellation resource group. +2. Select the `scale-set-controlplanes`. +3. Go to **settings** and **scaling**. +4. Set the new (increased) **instance count** and **save**. + + + + +1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). +2. **Edit** the **control-plane** instance group. +3. Set the new (increased) **number of instances** and **save**. + + + + +Dynamic cluster scaling isn't yet supported for STACKIT. +Support will be introduced in one of the upcoming releases. + + + + +If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.23/workflows/storage.md b/docs/versioned_docs/version-2.23/workflows/storage.md new file mode 100644 index 000000000..a5c52be90 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/storage.md @@ -0,0 +1,281 @@ +# Use persistent storage + +Persistent storage in Kubernetes requires cloud-specific configuration. +For abstraction of container storage, Kubernetes offers [volumes](https://kubernetes.io/docs/concepts/storage/volumes/), +allowing users to mount storage solutions directly into containers. +The [Container Storage Interface (CSI)](https://kubernetes-csi.github.io/docs/) is the standard interface for exposing arbitrary block and file storage systems into containers in Kubernetes. +Cloud service providers (CSPs) offer their own CSI-based solutions for cloud storage. + +## Confidential storage + +Most cloud storage solutions support encryption, such as [GCE Persistent Disks (PD)](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek). +Constellation supports the available CSI-based storage options for Kubernetes engines in AWS, Azure, GCP, and STACKIT. +However, their encryption takes place in the storage backend and is managed by the CSP. +Thus, using the default CSI drivers for these storage types means trusting the CSP with your persistent data. + +To address this, Constellation provides CSI drivers for AWS EBS, Azure Disk, GCE PD, and OpenStack Cinder, offering [encryption on the node level](../architecture/keys.md#storage-encryption). They enable transparent encryption for persistent volumes without needing to trust the cloud backend. Plaintext data never leaves the confidential VM context, offering you confidential storage. + +For more details see [encrypted persistent storage](../architecture/encrypted-storage.md). + +## CSI drivers + +Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. + + + + +**Constellation CSI driver for AWS Elastic Block Store** +Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. + + + + +**Constellation CSI driver for Azure Disk**: +Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. +See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. +Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. + + + + +**Constellation CSI driver for GCP Persistent Disk**: +Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. + + + + +**Constellation CSI driver for STACKIT / OpenStack Cinder** +Mount [Cinder](https://docs.openstack.org/cinder/latest/) block storage volumes into your Constellation cluster. +Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-cloud-provider-openstack) for more information. + + + + +Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. + +## Installation + +The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. +If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. + + + + +AWS comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [SSDs of `gp3` type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +Azure comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [Standard SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [Premium SSDs](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +GCP comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [standard persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [performance (SSD) persistent disks](https://cloud.google.com/compute/docs/disks#pdspecs) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +STACKIT comes with two storage classes by default. + +* `encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk +* `integrity-encrypted-rwo` + * Uses [disks of `storage_premium_perf1` type](https://docs.stackit.cloud/stackit/en/service-plans-blockstorage-75137974.html) + * ext-4 filesystem + * Encryption of all data written to disk + * Integrity protection of data written to disk + +For more information on encryption algorithms and key sizes, refer to [cryptographic algorithms](../architecture/encrypted-storage.md#cryptographic-algorithms). + +:::info + +The default storage class is set to `encrypted-rwo` for performance reasons. +If you want integrity-protected storage, set the `storageClassName` parameter of your persistent volume claim to `integrity-encrypted-rwo`. + +Alternatively, you can create your own storage class with integrity protection enabled by adding `csi.storage.k8s.io/fstype: ext4-integrity` to the class `parameters`. +Or use another filesystem by specifying another file system type with the suffix `-integrity`, e.g., `csi.storage.k8s.io/fstype: xfs-integrity`. + +Note that volume expansion isn't supported for integrity-protected disks. + +::: + + + + +1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + + A [persistent volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) is a request for storage with certain properties. + It can refer to a storage class. + The following creates a persistent volume claim, requesting 20 GB of storage via the `encrypted-rwo` storage class: + + ```bash + cat < + +--- + +You can terminate your cluster using the CLI. For this, you need the Terraform state directory named [`constellation-terraform`](../reference/terraform.md) in the current directory. + +:::danger + +All ephemeral storage and state of your cluster will be lost. Make sure any data is safely stored in persistent storage. Constellation can recreate your cluster and the associated encryption keys, but won't backup your application data automatically. + +::: + + + +Terminate the cluster by running: + +```bash +constellation terminate +``` + +Or without confirmation (e.g., for automation purposes): + +```bash +constellation terminate --yes +``` + +This deletes all resources created by Constellation in your cloud environment. +All local files created by the `apply` command are deleted as well, except for `constellation-mastersecret.json` and the configuration file. + +:::caution + +Termination can fail if additional resources have been created that depend on the ones managed by Constellation. In this case, you need to delete these additional +resources manually. Just run the `terminate` command again afterward to continue the termination process of the cluster. + +::: + + + +Terminate the cluster by running: + +```bash +terraform destroy +``` + +Delete all files that are no longer needed: + +```bash +rm constellation-state.yaml constellation-admin.conf +``` + +Only the `constellation-mastersecret.json` and the configuration file remain. + + + diff --git a/docs/versioned_docs/version-2.23/workflows/terraform-provider.md b/docs/versioned_docs/version-2.23/workflows/terraform-provider.md new file mode 100644 index 000000000..c7a795d3f --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/terraform-provider.md @@ -0,0 +1,140 @@ +# Use the Terraform provider + +The Constellation Terraform provider allows to manage the full lifecycle of a Constellation cluster (namely creation, upgrades, and deletion) via Terraform. +The provider is available through the [Terraform registry](https://registry.terraform.io/providers/edgelesssys/constellation/latest) and is released in lock-step with Constellation releases. + +## Prerequisites + +- a Linux / Mac operating system (ARM64/AMD64) +- a Terraform installation of version `v1.4.4` or above + +## Quick setup + +This example shows how to set up a Constellation cluster with the reference IAM and infrastructure setup. This setup is also used when creating a Constellation cluster through the Constellation CLI. You can either consume the IAM / infrastructure modules through a remote source (recommended) or local files. The latter requires downloading the infrastructure and IAM modules for the corresponding CSP from `terraform-modules.zip` on the [Constellation release page](https://github.com/edgelesssys/constellation/releases/latest) and placing them in the Terraform workspace directory. + +1. Create a directory (workspace) for your Constellation cluster. + + ```bash + mkdir constellation-workspace + cd constellation-workspace + ``` + +2. Use one of the [example configurations for using the Constellation Terraform provider](https://github.com/edgelesssys/constellation/tree/main/terraform-provider-constellation/examples/full) or create a `main.tf` file and fill it with the resources you want to create. The [Constellation Terraform provider documentation](https://registry.terraform.io/providers/edgelesssys/constellation/latest) offers thorough documentation on the resources and their attributes. +3. Initialize and apply the Terraform configuration. + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +:::info +On SEV-SNP, you need to manually patch the policy of the MAA provider before creating the Constellation cluster, as this feature isn't available in Azure's Terraform provider yet. The Constellation CLI provides a utility for patching, but you can also do it manually. + + ```bash + terraform init + terraform apply -target module.azure_iam # adjust resource path if not using the example configuration + terraform apply -target module.azure_infrastructure # adjust resource path if not using the example configuration + constellation maa-patch $(terraform output -raw maa_url) # adjust output path / input if not using the example configuration or manually patch the resource + terraform apply -target constellation_cluster.azure_example # adjust resource path if not using the example configuration + ``` + + Use the following policy if manually performing the patch. + + ``` + version= 1.0; + authorizationrules + { + [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny(); + [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny(); + // The line below was edited to use the MAA provider within Constellation. Do not edit manually. + //[type=="secureboot", value==false] => deny(); + [type=="x-ms-azurevm-signingdisabled", value==false] => deny(); + [type=="x-ms-azurevm-dbvalidated", value==false] => deny(); + [type=="x-ms-azurevm-dbxvalidated", value==false] => deny(); + => permit(); + }; + issuancerules + { + }; + ``` + +::: + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + Initialize the providers and apply the configuration. + + ```bash + terraform init + terraform apply + ``` + + Optionally, you can prefix the `terraform apply` command with `TF_LOG=INFO` to collect [Terraform logs](https://developer.hashicorp.com/terraform/internals/debugging) while applying the configuration. This may provide helpful output in debugging scenarios. + + + +4. Connect to the cluster. + + ```bash + terraform output -raw kubeconfig > constellation-admin.conf + export KUBECONFIG=$(realpath constellation-admin.conf) + ``` + +## Bringing your own infrastructure + +Instead of using the example infrastructure used in the [quick setup](#quick-setup), you can also provide your own infrastructure. +If you need a starting point for a custom infrastructure setup, you can download the infrastructure / IAM Terraform modules for the respective CSP from the Constellation [GitHub releases](https://github.com/edgelesssys/constellation/releases). You can modify and extend the modules per your requirements, while keeping the basic functionality intact. +The module contains: + +- `{csp}`: cloud resources the cluster runs on +- `iam/{csp}`: IAM resources used within the cluster + +When upgrading your cluster, make sure to check the Constellation release notes for potential breaking changes in the reference infrastructure / IAM modules that need to be considered. + +## Cluster upgrades + +:::tip +Also see the [general documentation on cluster upgrades](./upgrade.md). +::: + +The steps for applying the upgrade are as follows: + +1. Update the version constraint of the Constellation Terraform provider in the `required_providers` block in your Terraform configuration. +2. If you explicitly set any of the version attributes of the provider's resources and data sources (e.g. `image_version` or `constellation_microservice_version`), make sure to update them too. Refer to Constellation's [version support policy](https://github.com/edgelesssys/constellation/blob/main/dev-docs/workflows/versions-support.md) for more information on how each Constellation version and its dependencies are supported. +3. Update the IAM / infrastructure configuration. + - For [remote addresses as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#fetching-archives-over-http), update the version number inside the address of the `source` field of the infrastructure / IAM module to the target version. + - For [local paths as module sources](https://developer.hashicorp.com/terraform/language/modules/sources#local-paths) or when [providing your own infrastructure](#bringing-your-own-infrastructure), see the changes made in the reference modules since the upgrade's origin version and adjust your infrastructure / IAM configuration accordingly. +4. Upgrade the Terraform module and provider dependencies and apply the targeted configuration. + +```bash + terraform init -upgrade + terraform apply +``` diff --git a/docs/versioned_docs/version-2.23/workflows/troubleshooting.md b/docs/versioned_docs/version-2.23/workflows/troubleshooting.md new file mode 100644 index 000000000..903c829e0 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/troubleshooting.md @@ -0,0 +1,200 @@ +# Troubleshooting + +This section aids you in finding problems when working with Constellation. + +## Common issues + +### Issues with creating new clusters + +When you create a new cluster, you should always use the [latest release](https://github.com/edgelesssys/constellation/releases/latest). +If something doesn't work, check out the [known issues](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22). + +### Azure: Resource Providers can't be registered + +On Azure, you may receive the following error when running `apply` or `terminate` with limited IAM permissions: + +```shell-session +Error: Error ensuring Resource Providers are registered. + +Terraform automatically attempts to register the Resource Providers it supports to +ensure it's able to provision resources. + +If you don't have permission to register Resource Providers you may wish to use the +"skip_provider_registration" flag in the Provider block to disable this functionality. + +[...] +``` + +To continue, please ensure that the [required resource providers](../getting-started/install.md#required-permissions) have been registered in your subscription by your administrator. + +Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `apply` or `terminate` again. +For example: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation apply +``` + +Or alternatively, for `terminate`: + +```bash +ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate +``` + +### Azure: Can't update attestation policy + +On Azure, you may receive the following error when running `apply` from within an Azure environment, e.g., an Azure VM: + +```shell-session +An error occurred: patching policies: updating attestation policy: unexpected status code: 403 Forbidden +``` + +The problem occurs because the Azure SDK we use internally attempts to [authenticate towards the Azure API with the managed identity of your current environment instead of the Azure CLI token](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential). + +We decided not to deviate from this behavior and comply with the ordering of credentials. + +A solution is to add the [required permissions](../getting-started/install.md#required-permissions) to the managed identity of your environment. For example, the managed identity of your Azure VM, instead of the account that you've authenticated with in the Azure CLI. + +If your setup requires a change in the ordering of credentials, please open an issue and explain your desired behavior. + + + +### Nodes fail to join with error `untrusted measurement value` + +This error indicates that a node's [attestation statement](../architecture/attestation.md) contains measurements that don't match the trusted values expected by the [JoinService](../architecture/microservices.md#joinservice). +This may for example happen if the cloud provider updates the VM's firmware such that it influences the [runtime measurements](../architecture/attestation.md#runtime-measurements) in an unforeseen way. +A failed upgrade due to an erroneous attestation config can also cause this error. +You can change the expected measurements to resolve the failure. + +:::caution + +Attestation and trusted measurements are crucial for the security of your cluster. +Be extra careful when manually changing these settings. +When in doubt, check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +:::tip + +During an upgrade with modified attestation config, a backup of the current configuration is stored in the `join-config` config map in the `kube-system` namespace under the `attestationConfig_backup` key. To restore the old attestation config after a failed upgrade, replace the value of `attestationConfig` with the value from `attestationConfig_backup`: + +```bash +kubectl patch configmaps -n kube-system join-config -p "{\"data\":{\"attestationConfig\":\"$(kubectl get configmaps -n kube-system join-config -o "jsonpath={.data.attestationConfig_backup}")\"}}" +``` + +::: + +You can use the `apply` command to change measurements of a running cluster: + +1. Modify the `measurements` key in your local `constellation-conf.yaml` to the expected values. +2. Run `constellation apply`. + +Keep in mind that running `apply` also applies any version changes from your config to the cluster. + +You can run these commands to learn about the versions currently configured in the cluster: + +- Kubernetes API server version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.kubernetesClusterVersion` +- image version: `kubectl get nodeversion constellation-version -o json -n kube-system | jq .spec.imageVersion` +- microservices versions: `helm list --filter 'constellation-services' -n kube-system` + +### Upgrading Kubernetes resources fails + +Constellation manages its Kubernetes resources using Helm. +When applying an upgrade, the charts that are about to be installed, and a values override file `overrides.yaml`, +are saved to disk in your current workspace under `constellation-upgrade/upgrade-/helm-charts/`. +If upgrading the charts using the Constellation CLI fails, you can review these charts and try to manually apply the upgrade. + +:::caution + +Changing and manually applying the charts may destroy cluster resources and can lead to broken Constellation deployments. +Proceed with caution and when in doubt, +check if the encountered [issue is known](https://github.com/edgelesssys/constellation/issues?q=is%3Aopen+is%3Aissue+label%3A%22known+issue%22) or [contact support](https://github.com/edgelesssys/constellation#support). + +::: + +## Diagnosing issues + +### Logs + +To get started on diagnosing issues with Constellation, it's often helpful to collect logs from nodes, pods, or other resources in the cluster. Most logs are available through Kubernetes' standard +[logging interfaces](https://kubernetes.io/docs/concepts/cluster-administration/logging/). + +To debug issues occurring at boot time of the nodes, you can use the serial console interface of the CSP while the machine boots to get a read-only view of the boot logs. + +Apart from that, Constellation also offers further [observability integrations](../architecture/observability.md). + +### Node shell access + +Debugging via a shell on a node is [directly supported by Kubernetes](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#node-shell-session). + +1. Figure out which node to connect to: + + ```bash + kubectl get nodes + # or to see more information, such as IPs: + kubectl get nodes -o wide + ``` + +2. Connect to the node: + + ```bash + kubectl debug node/constell-worker-xksa0-000000 -it --image=busybox + ``` + + You will be presented with a prompt. + + The nodes file system is mounted at `/host`. + +3. Once finished, clean up the debug pod: + + ```bash + kubectl delete pod node-debugger-constell-worker-xksa0-000000-bjthj + ``` + +### Emergency SSH access + +Emergency SSH access to nodes can be useful to diagnose issues or download important data even if the Kubernetes API isn't reachable anymore. + +1. Enter the `constellation-terraform` directory in your Constellation workspace and enable emergency SSH access to the cluster: + + ```bash + cd constellation-terraform + echo "emergency_ssh = true" >> ./terraform.tfvars + terraform apply + ``` + +2. Sign an existing SSH key with your master secret: + + ```bash + cd ../ # go back to your Constellation workspace + constellation ssh --key your_public_key.pub + ``` + + A certificate is written to `constellation_cert.pub`. + + The certificate is valid for 24 hours and enables you to access your Constellation nodes using + [certificate based authentication](https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Certificate-based_Authentication). + +3. Now you can connect to any Constellation node using your certificate and your private key. + + ```bash + ssh -o CertificateFile=constellation_cert.pub -i root@ + ``` + + Normally, you don't have access to the Constellation nodes since they reside in a private network. + To access those nodes anyways, you can use your Constellation load balancer as a proxy jump host. + For this, use something along the following SSH client configuration: + + ```text + Host + ProxyJump none + + Host * + IdentityFile + PreferredAuthentications publickey + CertificateFile=constellation_cert.pub + User root + ProxyJump + ``` + + With this configuration you can connect to a Constellation node using `ssh -F `. + You can obtain the private node IP and the domain name of the load balancer using your CSP's web UI. diff --git a/docs/versioned_docs/version-2.23/workflows/trusted-launch.md b/docs/versioned_docs/version-2.23/workflows/trusted-launch.md new file mode 100644 index 000000000..d6d01d8eb --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/trusted-launch.md @@ -0,0 +1,54 @@ +# Use Azure trusted launch VMs + +Constellation also supports [trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch) on Microsoft Azure. Trusted launch VMs don't offer the same level of security as Confidential VMs, but are available in more regions and in larger quantities. The main difference between trusted launch VMs and normal VMs is that the former offer vTPM-based remote attestation. When used with trusted launch VMs, Constellation relies on vTPM-based remote attestation to verify nodes. + +:::caution + +Trusted launch VMs don't provide runtime encryption and don't keep the cloud service provider (CSP) out of your trusted computing base. + +::: + +Constellation supports trusted launch VMs with instance types `Standard_D*_v4` and `Standard_E*_v4`. Run `constellation config instance-types` for a list of all supported instance types. + +## VM images + +Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. + +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. + +After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. +You can use a script to do this: + +```bash +wget https://raw.githubusercontent.com/edgelesssys/constellation/main/hack/importAzure.sh +chmod +x importAzure.sh +AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_IMAGE_FILE=./constellation.img ./importAzure.sh +``` + +The script creates the following resources: + +1. A new image gallery with the default name `constellation-import` +2. A new image definition with the default name `constellation` +3. The actual image with the provided version. In this case `2.2.0` + +Once the import is completed, use the `ID` of the image version in your `constellation-conf.yaml` for the `image` field. Set `confidentialVM` to `false`. + +Fetch the image measurements: + +```bash +IMAGE_VERSION=2.2.0 +URL=https://public-edgeless-constellation.s3.us-east-2.amazonaws.com//communitygalleries/constellationcvm-b3782fa0-0df7-4f2f-963e-fc7fc42663df/images/constellation/versions/$IMAGE_VERSION/measurements.yaml +constellation config fetch-measurements -u$URL -s$URL.sig +``` + +:::info + +The [`constellation apply`](create.md) command will issue a warning because manually imported images aren't recognized as production grade images: + +```shell-session +Configured image doesn't look like a released production image. Double check image before deploying to production. +``` + +Please ignore this warning. + +::: diff --git a/docs/versioned_docs/version-2.23/workflows/upgrade.md b/docs/versioned_docs/version-2.23/workflows/upgrade.md new file mode 100644 index 000000000..3db2ecad6 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/upgrade.md @@ -0,0 +1,110 @@ +# Upgrade your cluster + +Constellation provides an easy way to upgrade all components of your cluster, without disrupting its availability. +Specifically, you can upgrade the Kubernetes version, the nodes' image, and the Constellation microservices. +You configure the desired versions in your local Constellation configuration and trigger upgrades with the `apply` command. +To learn about available versions you use the `upgrade check` command. +Which versions are available depends on the CLI version you are using. + +## Update the CLI + +Each CLI comes with a set of supported microservice and Kubernetes versions. +Most importantly, a given CLI version can only upgrade a cluster of the previous minor version, but not older ones. +This means that you have to upgrade your CLI and cluster one minor version at a time. + +For example, if you are currently on CLI version v2.6 and the latest version is v2.8, you should + +* upgrade the CLI to v2.7, +* upgrade the cluster to v2.7, +* and only then continue upgrading the CLI (and the cluster) to v2.8 after. + +Also note that if your current Kubernetes version isn't supported by the next CLI version, use your current CLI to upgrade to a newer Kubernetes version first. + +To learn which Kubernetes versions are supported by a particular CLI, run [constellation config kubernetes-versions](../reference/cli.md#constellation-config-kubernetes-versions). + +## Migrate the configuration + +The Constellation configuration file is located in the file `constellation-conf.yaml` in your workspace. +Refer to the [migration reference](../reference/migration.md) to check if you need to update fields in your configuration file. +Use [`constellation config migrate`](../reference/cli.md#constellation-config-migrate) to automatically update an old config file to a new format. + +## Check for upgrades + +To learn which versions the current CLI can upgrade to and what's installed in your cluster, run: + +```bash +# Show possible upgrades +constellation upgrade check + +# Show possible upgrades and write them to config file +constellation upgrade check --update-config +``` + +You can either enter the reported target versions into your config manually or run the above command with the `--update-config` flag. +When using this flag, the `kubernetesVersion`, `image`, `microserviceVersion`, and `attestation` fields are overwritten with the smallest available upgrade. + +## Apply the upgrade + +Once you updated your config with the desired versions, you can trigger the upgrade with this command: + +```bash +constellation apply +``` + +Microservice upgrades will be finished within a few minutes, depending on the cluster size. +If you are interested, you can monitor pods restarting in the `kube-system` namespace with your tool of choice. + +Image and Kubernetes upgrades take longer. +For each node in your cluster, a new node has to be created and joined. +The process usually takes up to ten minutes per node. + +When applying an upgrade, the Helm charts for the upgrade as well as backup files of Constellation-managed Custom Resource Definitions, Custom Resources, and Terraform state are created. +You can use the Terraform state backup to restore previous resources in case an upgrade misconfigured or erroneously deleted a resource. +You can use the Custom Resource (Definition) backup files to restore Custom Resources and Definitions manually (e.g., via `kubectl apply`) if the automatic migration of those resources fails. +You can use the Helm charts to manually apply upgrades to the Kubernetes resources, should an upgrade fail. + +:::note + +For advanced users: the upgrade consists of several phases that can be individually skipped through the `--skip-phases` flag. +The phases are `infrastracture` for the cloud resource management through Terraform, `helm` for the chart management of the microservices, `image` for OS image upgrades, and `k8s` for Kubernetes version upgrades. + +::: + +## Check the status + +Upgrades are asynchronous operations. +After you run `apply`, it will take a while until the upgrade has completed. +To understand if an upgrade is finished, you can run: + +```bash +constellation status +``` + +This command displays the following information: + +* The installed services and their versions +* The image and Kubernetes version the cluster is expecting on each node +* How many nodes are up to date + +Here's an example output: + +```shell-session +Target versions: + Image: v2.6.0 + Kubernetes: v1.25.8 +Service versions: + Cilium: v1.12.1 + cert-manager: v1.10.0 + constellation-operators: v2.6.0 + constellation-services: v2.6.0 +Cluster status: Some node versions are out of date + Image: 23/25 + Kubernetes: 25/25 +``` + +This output indicates that the cluster is running Kubernetes version `1.25.8`, and all nodes have the appropriate binaries installed. +23 out of 25 nodes have already upgraded to the targeted image version of `2.6.0`, while two are still in progress. + +## Apply further upgrades + +After the upgrade is finished, you can run `constellation upgrade check` again to see if there are more upgrades available. If so, repeat the process. diff --git a/docs/versioned_docs/version-2.23/workflows/verify-cli.md b/docs/versioned_docs/version-2.23/workflows/verify-cli.md new file mode 100644 index 000000000..e33569d37 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/verify-cli.md @@ -0,0 +1,129 @@ +# Verify the CLI + +:::info +This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. +::: + + + +--- + +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. + +:::note +The public key for Edgeless Systems' long-term code-signing key is: + +``` +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEf8F1hpmwE+YCFXzjGtaQcrL6XZVT +JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== +-----END PUBLIC KEY----- +``` + +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +::: + +The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. + +You should always ensure that (1) your CLI executable was signed with the private key corresponding to the above public key and that (2) there is a corresponding entry in the Rekor transparency log. Both can be done as described in the following. + +:::info +You don't need to verify the Constellation node images. This is done automatically by your CLI and the rest of Constellation. +::: + +## Verify the signature + +:::info +This guide assumes Linux on an amd64 processor. The exact steps for other platforms differ slightly. +::: + +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: + +```shell-session +$ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +Verified OK +``` + +The above performs an offline verification of the provided public key, signature, and executable. To also verify that a corresponding entry exists in the public Rekor transparency log, add the variable `COSIGN_EXPERIMENTAL=1`: + +```shell-session +$ COSIGN_EXPERIMENTAL=1 cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 + +tlog entry verified with uuid: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 index: 3477047 +Verified OK +``` + +🏁 You now know that your CLI executable was officially released and signed by Edgeless Systems. + +### Optional: Manually inspect the transparency log + +To further inspect the public Rekor transparency log, [install the Rekor CLI](https://docs.sigstore.dev/logging/installation). A search for the CLI executable should give a single UUID. (Note that this UUID contains the UUID from the previous `cosign` command.) + +```shell-session +$ rekor-cli search --artifact constellation-linux-amd64 + +Found matching entries (listed by UUID): +362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +``` + +With this UUID you can get the full entry from the transparency log: + +```shell-session +$ rekor-cli get --uuid=362f8ecba72f4326afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 + +LogID: c0d23d6ad406973f9559f3ba2d1ca01f84147d8ffc5b8445c224f98b9591801d +Index: 3477047 +IntegratedTime: 2022-09-12T22:28:16Z +UUID: afaba7f6635b3e058888692841848e5514357315be9528474b23f5dcccb82b13 +Body: { + "HashedRekordObj": { + "data": { + "hash": { + "algorithm": "sha256", + "value": "40e137b9b9b8204d672642fd1e181c6d5ccb50cfc5cc7fcbb06a8c2c78f44aff" + } + }, + "signature": { + "content": "MEUCIQCSER3mGj+j5Pr2kOXTlCIHQC3gT30I7qkLr9Awt6eUUQIgcLUKRIlY50UN8JGwVeNgkBZyYD8HMxwC/LFRWoMn180=", + "publicKey": { + "content": "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFZjhGMWhwbXdFK1lDRlh6akd0YVFjckw2WFpWVApKbUVlNWlTTHZHMVN5UVNBZXc3V2RNS0Y2bzl0OGUyVEZ1Q2t6bE9oaGx3czJPSFdiaUZabkZXQ0Z3PT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + } + } + } +} +``` + +The field `publicKey` should contain Edgeless Systems' public key in Base64 encoding. + +You can get an exhaustive list of artifact signatures issued by Edgeless Systems via the following command: + +```bash +rekor-cli search --public-key https://edgeless.systems/es.pub --pki-format x509 +``` + +Edgeless Systems monitors this list to detect potential unauthorized use of its private key. + +## Verify the provenance + +Provenance attests that a software artifact was produced by a specific repository and build system invocation. For more information on provenance visit [slsa.dev](https://slsa.dev/provenance/v0.2) and learn about the [adoption of SLSA for Constellation](../reference/slsa.md). + +Just as checking its signature proves that the CLI hasn't been manipulated, checking the provenance proves that the artifact was produced by the expected build process and hasn't been tampered with. + +To verify the provenance, first install the [slsa-verifier](https://github.com/slsa-framework/slsa-verifier). Then make sure you have the provenance file (`constellation.intoto.jsonl`) and Constellation CLI downloaded. Both are available on the [GitHub release page](https://github.com/edgelesssys/constellation/releases). + +:::info +The same provenance file is valid for all Constellation CLI executables of a given version independent of the target platform. +::: + +Use the verifier to perform the check: + +```shell-session +$ slsa-verifier verify-artifact constellation-linux-amd64 \ + --provenance-path constellation.intoto.jsonl \ + --source-uri github.com/edgelesssys/constellation + +Verified signature against tlog entry index 7771317 at URL: https://rekor.sigstore.dev/api/v1/log/entries/24296fb24b8ad77af2c04c8b4ae0d5bc5... +Verified build using builder https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@refs/tags/v1.2.2 at commit 18e9924b416323c37b9cdfd6cc728de8a947424a +PASSED: Verified SLSA provenance +``` diff --git a/docs/versioned_docs/version-2.23/workflows/verify-cluster.md b/docs/versioned_docs/version-2.23/workflows/verify-cluster.md new file mode 100644 index 000000000..b6595ebf2 --- /dev/null +++ b/docs/versioned_docs/version-2.23/workflows/verify-cluster.md @@ -0,0 +1,97 @@ +# Verify your cluster + +Constellation's [attestation feature](../architecture/attestation.md) allows you, or a third party, to verify the integrity and confidentiality of your Constellation cluster. + +## Fetch measurements + +To verify the integrity of Constellation you need trusted measurements to verify against. For each node image released by Edgeless Systems, there are signed measurements, which you can download using the CLI: + +```bash +constellation config fetch-measurements +``` + +This command performs the following steps: + +1. Download the signed measurements for the configured image. By default, this will use Edgeless Systems' public measurement registry. +2. Verify the signature of the measurements. This will use Edgeless Systems' [public key](https://edgeless.systems/es.pub). +3. Write measurements into configuration file. + +The configuration file then contains a list of `measurements` similar to the following: + +```yaml +# ... +measurements: + 0: + expected: "0f35c214608d93c7a6e68ae7359b4a8be5a0e99eea9107ece427c4dea4e439cf" + warnOnly: false + 4: + expected: "02c7a67c01ec70ffaf23d73a12f749ab150a8ac6dc529bda2fe1096a98bf42ea" + warnOnly: false + 5: + expected: "e6949026b72e5045706cd1318889b3874480f7a3f7c5c590912391a2d15e6975" + warnOnly: true + 8: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 9: + expected: "f0a6e8601b00e2fdc57195686cd4ef45eb43a556ac1209b8e25d993213d68384" + warnOnly: false + 11: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 12: + expected: "da99eb6cf7c7fbb692067c87fd5ca0b7117dc293578e4fea41f95d3d3d6af5e2" + warnOnly: false + 13: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false + 14: + expected: "d7c4cc7ff7933022f013e03bdee875b91720b5b86cf1753cad830f95e791926f" + warnOnly: true + 15: + expected: "0000000000000000000000000000000000000000000000000000000000000000" + warnOnly: false +# ... +``` + +Each entry specifies the expected value of the Constellation node, and whether the measurement should be enforced (`warnOnly: false`), or only a warning should be logged (`warnOnly: true`). +By default, the subset of the [available measurements](../architecture/attestation.md#runtime-measurements) that can be locally reproduced and verified is enforced. + +During attestation, the validating side (CLI or [join service](../architecture/microservices.md#joinservice)) compares each measurement reported by the issuing side (first node or joining node) individually. +For mismatching measurements that have set `warnOnly` to `true` only a warning is emitted. +For mismatching measurements that have set `warnOnly` to `false` an error is emitted and attestation fails. +If attestation fails for a new node, it isn't permitted to join the cluster. + +## The *verify* command + +:::note +The steps below are purely optional. They're automatically executed by `constellation apply` when you initialize your cluster. The `constellation verify` command mostly has an illustrative purpose. +::: + +The `verify` command obtains and verifies an attestation statement from a running Constellation cluster. + +```bash +constellation verify [--cluster-id ...] +``` + +From the attestation statement, the command verifies the following properties: + +* The cluster is using the correct Confidential VM (CVM) type. +* Inside the CVMs, the correct node images are running. The node images are identified through the measurements obtained in the previous step. +* The unique ID of the cluster matches the one from your `constellation-state.yaml` file or passed in via `--cluster-id`. + +Once the above properties are verified, you know that you are talking to the right Constellation cluster and it's in a good and trustworthy shape. + +### Custom arguments + +The `verify` command also allows you to verify any Constellation deployment that you have network access to. For this you need the following: + +* The IP address of a running Constellation cluster's [VerificationService](../architecture/microservices.md#verificationservice). The `VerificationService` is exposed via a `NodePort` service using the external IP address of your cluster. Run `kubectl get nodes -o wide` and look for `EXTERNAL-IP`. +* The cluster's *clusterID*. See [cluster identity](../architecture/keys.md#cluster-identity) for more details. +* A `constellation-conf.yaml` file with the expected measurements of the cluster in your working directory. + +For example: + +```shell-session +constellation verify -e 192.0.2.1 --cluster-id Q29uc3RlbGxhdGlvbkRvY3VtZW50YXRpb25TZWNyZXQ= +``` diff --git a/docs/versioned_docs/version-2.3/architecture/attestation.md b/docs/versioned_docs/version-2.3/architecture/attestation.md index f335038f6..28e8e62cf 100644 --- a/docs/versioned_docs/version-2.3/architecture/attestation.md +++ b/docs/versioned_docs/version-2.3/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,8 +217,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.3/architecture/keys.md b/docs/versioned_docs/version-2.3/architecture/keys.md index aa4e35496..b7d7ef6f5 100644 --- a/docs/versioned_docs/version-2.3/architecture/keys.md +++ b/docs/versioned_docs/version-2.3/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.3/getting-started/first-steps.md b/docs/versioned_docs/version-2.3/getting-started/first-steps.md index a749ca6a9..d1cd06cf6 100644 --- a/docs/versioned_docs/version-2.3/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.3/getting-started/first-steps.md @@ -11,29 +11,29 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step 1. Create the configuration file for your selected cloud provider. - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in your current working directory. @@ -41,9 +41,9 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step First you need to create an [IAM configuration](../workflows/config.md#creating-an-iam-configuration). The easiest way to do this is the following CLI command: - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest @@ -57,21 +57,21 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step * `northeurope` * `westeurope` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. - + - + ```bash constellation iam create aws --zone=eu-central-1a --prefix=constellTest @@ -88,8 +88,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + Now, fill the output values of the command into the corresponding fields of the `constellation-conf.yaml` file. diff --git a/docs/versioned_docs/version-2.3/getting-started/install.md b/docs/versioned_docs/version-2.3/getting-started/install.md index 91c4bb14e..36d8f541a 100644 --- a/docs/versioned_docs/version-2.3/getting-started/install.md +++ b/docs/versioned_docs/version-2.3/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,14 +105,15 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Compute` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` -* `microsoft.insights` + +- `Microsoft.Compute` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` +- `microsoft.insights` By default, Constellation tries to register these automatically if they haven't been registered before. @@ -127,8 +125,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -140,8 +138,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -272,8 +270,8 @@ such as `PowerUserAccess`, or use the following minimal set of permissions: Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -283,8 +281,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -300,8 +298,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -324,8 +322,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -341,10 +339,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.3/overview/clouds.md b/docs/versioned_docs/version-2.3/overview/clouds.md index 01e7a00c5..c48f23cf0 100644 --- a/docs/versioned_docs/version-2.3/overview/clouds.md +++ b/docs/versioned_docs/version-2.3/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.3/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.3/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.3/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.3/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.3/overview/product.md b/docs/versioned_docs/version-2.3/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.3/overview/product.md +++ b/docs/versioned_docs/version-2.3/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.3/workflows/config.md b/docs/versioned_docs/version-2.3/workflows/config.md index afd53812e..20aa0dada 100644 --- a/docs/versioned_docs/version-2.3/workflows/config.md +++ b/docs/versioned_docs/version-2.3/workflows/config.md @@ -6,62 +6,62 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the N2D family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all nitroTPM-enabled machines with a minimum of 4 vCPUs (`xlarge` or larger). Refer to the [list of nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html) or run `constellation config instance-types` to get the list of all supported options. - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. ## Creating an IAM configuration You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session. @@ -84,23 +84,23 @@ Paste the output into the corresponding fields of the `constellation-conf.yaml` Since `clientSecretValue` is a sensitive value, you can leave it empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session. ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session. @@ -122,16 +122,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -175,19 +175,19 @@ The following describes the configuration fields and how you obtain the required Since this is a sensitive value, alternatively you can leave `clientSecretValue` empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -201,9 +201,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -232,9 +232,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.3/workflows/recovery.md b/docs/versioned_docs/version-2.3/workflows/recovery.md index fd610fc67..0fd171036 100644 --- a/docs/versioned_docs/version-2.3/workflows/recovery.md +++ b/docs/versioned_docs/version-2.3/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.3/workflows/sbom.md b/docs/versioned_docs/version-2.3/workflows/sbom.md index ec9834b4f..e8ba25a64 100644 --- a/docs/versioned_docs/version-2.3/workflows/sbom.md +++ b/docs/versioned_docs/version-2.3/workflows/sbom.md @@ -15,7 +15,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -36,7 +36,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.3/workflows/scale.md b/docs/versioned_docs/version-2.3/workflows/scale.md index 3b7c0d479..bce045c66 100644 --- a/docs/versioned_docs/version-2.3/workflows/scale.md +++ b/docs/versioned_docs/version-2.3/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + :::caution @@ -72,8 +72,8 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + ## Control-plane node scaling @@ -81,24 +81,24 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + :::caution @@ -106,7 +106,7 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.3/workflows/storage.md b/docs/versioned_docs/version-2.3/workflows/storage.md index d0e5b188f..be9998676 100644 --- a/docs/versioned_docs/version-2.3/workflows/storage.md +++ b/docs/versioned_docs/version-2.3/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -57,8 +57,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -86,8 +86,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -115,8 +115,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + :::caution @@ -126,8 +126,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) @@ -186,8 +186,8 @@ The default storage class is responsible for all persistent volume claims that d Constellation creates a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -233,8 +233,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -280,8 +280,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -291,5 +291,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.3/workflows/troubleshooting.md b/docs/versioned_docs/version-2.3/workflows/troubleshooting.md index 3a28c9cd0..f948f5d06 100644 --- a/docs/versioned_docs/version-2.3/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.3/workflows/troubleshooting.md @@ -5,6 +5,7 @@ This section aids you in finding problems when working with Constellation. ## Azure: Resource Providers can't be registered On Azure, you may receive the following error when running `create` or `terminate` with limited IAM permissions: + ```shell-session Error: Error ensuring Resource Providers are registered. @@ -21,11 +22,13 @@ To continue, please ensure that the [required resource providers](../getting-sta Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `create` or `terminate` again. For example: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation create --control-plane-nodes 1 --worker-nodes 2 -y ``` Or alternatively, for `terminate`: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate ``` @@ -36,8 +39,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -47,8 +50,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -63,16 +66,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ## Connect to nodes diff --git a/docs/versioned_docs/version-2.3/workflows/trusted-launch.md b/docs/versioned_docs/version-2.3/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.3/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.3/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.3/workflows/verify-cli.md b/docs/versioned_docs/version-2.3/workflows/verify-cli.md index 4f6008cd0..01a2583d6 100644 --- a/docs/versioned_docs/version-2.3/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.3/workflows/verify-cli.md @@ -1,6 +1,6 @@ # Verify the CLI -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -12,7 +12,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -25,7 +25,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.4/architecture/attestation.md b/docs/versioned_docs/version-2.4/architecture/attestation.md index f335038f6..28e8e62cf 100644 --- a/docs/versioned_docs/version-2.4/architecture/attestation.md +++ b/docs/versioned_docs/version-2.4/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,8 +217,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.4/architecture/keys.md b/docs/versioned_docs/version-2.4/architecture/keys.md index aa4e35496..b7d7ef6f5 100644 --- a/docs/versioned_docs/version-2.4/architecture/keys.md +++ b/docs/versioned_docs/version-2.4/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.4/getting-started/first-steps.md b/docs/versioned_docs/version-2.4/getting-started/first-steps.md index 768e8dfcd..44e66ea95 100644 --- a/docs/versioned_docs/version-2.4/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.4/getting-started/first-steps.md @@ -11,29 +11,29 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step 1. Create the configuration file for your selected cloud provider. - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in your current working directory. @@ -41,9 +41,9 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step First you need to create an [IAM configuration](../workflows/config.md#creating-an-iam-configuration). The easiest way to do this is the following CLI command: - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest @@ -57,21 +57,21 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step * `northeurope` * `westeurope` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. - + - + ```bash constellation iam create aws --zone=eu-central-1a --prefix=constellTest @@ -88,8 +88,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + Now, fill the output values of the command into the corresponding fields of the `constellation-conf.yaml` file. diff --git a/docs/versioned_docs/version-2.4/getting-started/install.md b/docs/versioned_docs/version-2.4/getting-started/install.md index 91c4bb14e..36d8f541a 100644 --- a/docs/versioned_docs/version-2.4/getting-started/install.md +++ b/docs/versioned_docs/version-2.4/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,14 +105,15 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Compute` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` -* `microsoft.insights` + +- `Microsoft.Compute` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` +- `microsoft.insights` By default, Constellation tries to register these automatically if they haven't been registered before. @@ -127,8 +125,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -140,8 +138,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -272,8 +270,8 @@ such as `PowerUserAccess`, or use the following minimal set of permissions: Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -283,8 +281,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -300,8 +298,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -324,8 +322,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -341,10 +339,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.4/overview/clouds.md b/docs/versioned_docs/version-2.4/overview/clouds.md index 01e7a00c5..c48f23cf0 100644 --- a/docs/versioned_docs/version-2.4/overview/clouds.md +++ b/docs/versioned_docs/version-2.4/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.4/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.4/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.4/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.4/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.4/overview/product.md b/docs/versioned_docs/version-2.4/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.4/overview/product.md +++ b/docs/versioned_docs/version-2.4/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.4/workflows/config.md b/docs/versioned_docs/version-2.4/workflows/config.md index afd53812e..20aa0dada 100644 --- a/docs/versioned_docs/version-2.4/workflows/config.md +++ b/docs/versioned_docs/version-2.4/workflows/config.md @@ -6,62 +6,62 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the N2D family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all nitroTPM-enabled machines with a minimum of 4 vCPUs (`xlarge` or larger). Refer to the [list of nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html) or run `constellation config instance-types` to get the list of all supported options. - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. ## Creating an IAM configuration You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session. @@ -84,23 +84,23 @@ Paste the output into the corresponding fields of the `constellation-conf.yaml` Since `clientSecretValue` is a sensitive value, you can leave it empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session. ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session. @@ -122,16 +122,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -175,19 +175,19 @@ The following describes the configuration fields and how you obtain the required Since this is a sensitive value, alternatively you can leave `clientSecretValue` empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -201,9 +201,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -232,9 +232,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.4/workflows/recovery.md b/docs/versioned_docs/version-2.4/workflows/recovery.md index fd610fc67..0fd171036 100644 --- a/docs/versioned_docs/version-2.4/workflows/recovery.md +++ b/docs/versioned_docs/version-2.4/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.4/workflows/sbom.md b/docs/versioned_docs/version-2.4/workflows/sbom.md index ec9834b4f..e8ba25a64 100644 --- a/docs/versioned_docs/version-2.4/workflows/sbom.md +++ b/docs/versioned_docs/version-2.4/workflows/sbom.md @@ -15,7 +15,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -36,7 +36,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.4/workflows/scale.md b/docs/versioned_docs/version-2.4/workflows/scale.md index 3b7c0d479..bce045c66 100644 --- a/docs/versioned_docs/version-2.4/workflows/scale.md +++ b/docs/versioned_docs/version-2.4/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + :::caution @@ -72,8 +72,8 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + ## Control-plane node scaling @@ -81,24 +81,24 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + :::caution @@ -106,7 +106,7 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.4/workflows/storage.md b/docs/versioned_docs/version-2.4/workflows/storage.md index d0e5b188f..be9998676 100644 --- a/docs/versioned_docs/version-2.4/workflows/storage.md +++ b/docs/versioned_docs/version-2.4/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -57,8 +57,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -86,8 +86,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -115,8 +115,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + :::caution @@ -126,8 +126,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) @@ -186,8 +186,8 @@ The default storage class is responsible for all persistent volume claims that d Constellation creates a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -233,8 +233,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -280,8 +280,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -291,5 +291,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.4/workflows/troubleshooting.md b/docs/versioned_docs/version-2.4/workflows/troubleshooting.md index 3a28c9cd0..f948f5d06 100644 --- a/docs/versioned_docs/version-2.4/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.4/workflows/troubleshooting.md @@ -5,6 +5,7 @@ This section aids you in finding problems when working with Constellation. ## Azure: Resource Providers can't be registered On Azure, you may receive the following error when running `create` or `terminate` with limited IAM permissions: + ```shell-session Error: Error ensuring Resource Providers are registered. @@ -21,11 +22,13 @@ To continue, please ensure that the [required resource providers](../getting-sta Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `create` or `terminate` again. For example: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation create --control-plane-nodes 1 --worker-nodes 2 -y ``` Or alternatively, for `terminate`: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate ``` @@ -36,8 +39,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -47,8 +50,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -63,16 +66,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ## Connect to nodes diff --git a/docs/versioned_docs/version-2.4/workflows/trusted-launch.md b/docs/versioned_docs/version-2.4/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.4/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.4/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.4/workflows/verify-cli.md b/docs/versioned_docs/version-2.4/workflows/verify-cli.md index 4f6008cd0..01a2583d6 100644 --- a/docs/versioned_docs/version-2.4/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.4/workflows/verify-cli.md @@ -1,6 +1,6 @@ # Verify the CLI -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -12,7 +12,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -25,7 +25,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.5/architecture/attestation.md b/docs/versioned_docs/version-2.5/architecture/attestation.md index f335038f6..28e8e62cf 100644 --- a/docs/versioned_docs/version-2.5/architecture/attestation.md +++ b/docs/versioned_docs/version-2.5/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,8 +217,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.5/architecture/keys.md b/docs/versioned_docs/version-2.5/architecture/keys.md index aa4e35496..b7d7ef6f5 100644 --- a/docs/versioned_docs/version-2.5/architecture/keys.md +++ b/docs/versioned_docs/version-2.5/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.5/getting-started/first-steps.md b/docs/versioned_docs/version-2.5/getting-started/first-steps.md index 4e89bb0f2..9ce1d6be2 100644 --- a/docs/versioned_docs/version-2.5/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.5/getting-started/first-steps.md @@ -13,9 +13,9 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step First, you need to create a [configuration file](../workflows/config.md) and an [IAM configuration](../workflows/config.md#creating-an-iam-configuration). The easiest way to do this is the following CLI command: - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --generate-config @@ -29,21 +29,21 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step * `northeurope` * `westeurope` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --generate-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --generate-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. - + - + ```bash constellation iam create aws --zone=eu-central-1a --prefix=constellTest --generate-config @@ -60,8 +60,8 @@ If you don't have a cloud subscription, check out [MiniConstellation](first-step You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.5/getting-started/install.md b/docs/versioned_docs/version-2.5/getting-started/install.md index 91c4bb14e..36d8f541a 100644 --- a/docs/versioned_docs/version-2.5/getting-started/install.md +++ b/docs/versioned_docs/version-2.5/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,14 +105,15 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Compute` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` -* `microsoft.insights` + +- `Microsoft.Compute` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` +- `microsoft.insights` By default, Constellation tries to register these automatically if they haven't been registered before. @@ -127,8 +125,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -140,8 +138,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -272,8 +270,8 @@ such as `PowerUserAccess`, or use the following minimal set of permissions: Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -283,8 +281,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -300,8 +298,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -324,8 +322,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -341,10 +339,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.5/overview/clouds.md b/docs/versioned_docs/version-2.5/overview/clouds.md index dd31f866f..c48f23cf0 100644 --- a/docs/versioned_docs/version-2.5/overview/clouds.md +++ b/docs/versioned_docs/version-2.5/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.5/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.5/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.5/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.5/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.5/overview/product.md b/docs/versioned_docs/version-2.5/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.5/overview/product.md +++ b/docs/versioned_docs/version-2.5/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.5/workflows/config.md b/docs/versioned_docs/version-2.5/workflows/config.md index 100dedc8c..71f2d019d 100644 --- a/docs/versioned_docs/version-2.5/workflows/config.md +++ b/docs/versioned_docs/version-2.5/workflows/config.md @@ -6,29 +6,29 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. @@ -39,25 +39,25 @@ You can also automatically generate a configuration file by adding the `--genera ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the N2D family. Refer to [N2D machine series](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) or run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all nitroTPM-enabled machines with a minimum of 4 vCPUs (`xlarge` or larger). Refer to the [list of nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html) or run `constellation config instance-types` to get the list of all supported options. - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. @@ -66,8 +66,8 @@ Fill the desired VM type into the **instanceType** field in the `constellation-c You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you haven't generated a configuration file yet, you can do so by adding the `--generate-config` flag to the command. This creates a configuration file and populates it with the created IAM values. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session. @@ -90,23 +90,23 @@ Paste the output into the corresponding fields of the `constellation-conf.yaml` Since `clientSecretValue` is a sensitive value, you can leave it empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session. ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session. @@ -128,16 +128,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -181,19 +181,19 @@ The following describes the configuration fields and how you obtain the required Since this is a sensitive value, alternatively you can leave `clientSecretValue` empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -207,9 +207,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -238,9 +238,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.5/workflows/create.md b/docs/versioned_docs/version-2.5/workflows/create.md index 7fc991fd7..115effbdf 100644 --- a/docs/versioned_docs/version-2.5/workflows/create.md +++ b/docs/versioned_docs/version-2.5/workflows/create.md @@ -18,8 +18,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + Choose the initial size of your cluster. The following command creates a cluster with one control-plane and two worker nodes: @@ -32,8 +32,8 @@ For details on the flags, consult the command help via `constellation create -h` *create* stores your cluster's state into a [`terraform.tfstate`](../architecture/orchestration.md#cluster-creation-process) file in your workspace. - - + + Constellation supports managing the infrastructure via Terraform. This allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -68,8 +68,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step diff --git a/docs/versioned_docs/version-2.5/workflows/recovery.md b/docs/versioned_docs/version-2.5/workflows/recovery.md index fd610fc67..0fd171036 100644 --- a/docs/versioned_docs/version-2.5/workflows/recovery.md +++ b/docs/versioned_docs/version-2.5/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.5/workflows/sbom.md b/docs/versioned_docs/version-2.5/workflows/sbom.md index ec9834b4f..e8ba25a64 100644 --- a/docs/versioned_docs/version-2.5/workflows/sbom.md +++ b/docs/versioned_docs/version-2.5/workflows/sbom.md @@ -15,7 +15,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -36,7 +36,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.5/workflows/scale.md b/docs/versioned_docs/version-2.5/workflows/scale.md index 3b7c0d479..bce045c66 100644 --- a/docs/versioned_docs/version-2.5/workflows/scale.md +++ b/docs/versioned_docs/version-2.5/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + :::caution @@ -72,8 +72,8 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + ## Control-plane node scaling @@ -81,24 +81,24 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + :::caution @@ -106,7 +106,7 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.5/workflows/storage.md b/docs/versioned_docs/version-2.5/workflows/storage.md index d0e5b188f..be9998676 100644 --- a/docs/versioned_docs/version-2.5/workflows/storage.md +++ b/docs/versioned_docs/version-2.5/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -57,8 +57,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -86,8 +86,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -115,8 +115,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + :::caution @@ -126,8 +126,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) @@ -186,8 +186,8 @@ The default storage class is responsible for all persistent volume claims that d Constellation creates a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -233,8 +233,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -280,8 +280,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -291,5 +291,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.5/workflows/terminate.md b/docs/versioned_docs/version-2.5/workflows/terminate.md index a2556fe95..ee64a2784 100644 --- a/docs/versioned_docs/version-2.5/workflows/terminate.md +++ b/docs/versioned_docs/version-2.5/workflows/terminate.md @@ -8,8 +8,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -32,8 +32,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -48,5 +48,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.5/workflows/troubleshooting.md b/docs/versioned_docs/version-2.5/workflows/troubleshooting.md index 3a28c9cd0..f948f5d06 100644 --- a/docs/versioned_docs/version-2.5/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.5/workflows/troubleshooting.md @@ -5,6 +5,7 @@ This section aids you in finding problems when working with Constellation. ## Azure: Resource Providers can't be registered On Azure, you may receive the following error when running `create` or `terminate` with limited IAM permissions: + ```shell-session Error: Error ensuring Resource Providers are registered. @@ -21,11 +22,13 @@ To continue, please ensure that the [required resource providers](../getting-sta Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `create` or `terminate` again. For example: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation create --control-plane-nodes 1 --worker-nodes 2 -y ``` Or alternatively, for `terminate`: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate ``` @@ -36,8 +39,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -47,8 +50,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -63,16 +66,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ## Connect to nodes diff --git a/docs/versioned_docs/version-2.5/workflows/trusted-launch.md b/docs/versioned_docs/version-2.5/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.5/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.5/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.5/workflows/verify-cli.md b/docs/versioned_docs/version-2.5/workflows/verify-cli.md index 4f6008cd0..01a2583d6 100644 --- a/docs/versioned_docs/version-2.5/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.5/workflows/verify-cli.md @@ -1,6 +1,6 @@ # Verify the CLI -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -12,7 +12,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -25,7 +25,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.6/architecture/attestation.md b/docs/versioned_docs/version-2.6/architecture/attestation.md index 0c7a1487b..20f9909fd 100644 --- a/docs/versioned_docs/version-2.6/architecture/attestation.md +++ b/docs/versioned_docs/version-2.6/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,8 +217,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.6/architecture/keys.md b/docs/versioned_docs/version-2.6/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.6/architecture/keys.md +++ b/docs/versioned_docs/version-2.6/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.6/getting-started/first-steps.md b/docs/versioned_docs/version-2.6/getting-started/first-steps.md index ad89d89c8..df489f52a 100644 --- a/docs/versioned_docs/version-2.6/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.6/getting-started/first-steps.md @@ -17,9 +17,9 @@ If you encounter any problem with the following steps, make sure to use the [lat First, you need to create a [configuration file](../workflows/config.md) and an [IAM configuration](../workflows/config.md#creating-an-iam-configuration). The easiest way to do this is the following CLI command: - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --generate-config @@ -33,21 +33,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `northeurope` * `westeurope` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --generate-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --generate-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=eu-central-1a --prefix=constellTest --generate-config @@ -64,8 +64,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.6/getting-started/install.md b/docs/versioned_docs/version-2.6/getting-started/install.md index 91c4bb14e..36d8f541a 100644 --- a/docs/versioned_docs/version-2.6/getting-started/install.md +++ b/docs/versioned_docs/version-2.6/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,14 +105,15 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Compute` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` -* `microsoft.insights` + +- `Microsoft.Compute` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` +- `microsoft.insights` By default, Constellation tries to register these automatically if they haven't been registered before. @@ -127,8 +125,8 @@ You need the following permissions for your user account: If you don't have these permissions with scope *subscription*, ask your administrator to [create the service account and a resource group for your Constellation cluster](first-steps.md). Your user account needs the `Contributor` permission scoped to this resource group. - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. @@ -140,8 +138,8 @@ You need the following permissions on this project: Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -272,8 +270,8 @@ such as `PowerUserAccess`, or use the following minimal set of permissions: Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -283,8 +281,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -300,8 +298,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -324,8 +322,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -341,10 +339,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.6/overview/clouds.md b/docs/versioned_docs/version-2.6/overview/clouds.md index dd31f866f..c48f23cf0 100644 --- a/docs/versioned_docs/version-2.6/overview/clouds.md +++ b/docs/versioned_docs/version-2.6/overview/clouds.md @@ -24,11 +24,11 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. +The [CVMs available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. This impacts attestation capabilities. Currently, GCP doesn't offer CVM-based attestation at all. Instead, GCP provides attestation statements based on its regular [vTPM](https://cloud.google.com/blog/products/identity-security/virtual-trusted-platform-module-for-shielded-vms-security-in-plaintext), which is managed by the hypervisor. On GCP, the hypervisor is thus currently part of Constellation's TCB. ## Amazon Web Services (AWS) diff --git a/docs/versioned_docs/version-2.6/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.6/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.6/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.6/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.6/overview/performance.md b/docs/versioned_docs/version-2.6/overview/performance.md index 54f31019a..aef594c46 100644 --- a/docs/versioned_docs/version-2.6/overview/performance.md +++ b/docs/versioned_docs/version-2.6/overview/performance.md @@ -63,7 +63,6 @@ The following infrastructure configurations was used: - CVM: `false` - Zone: `europe-west3-b` - ### Results #### Network @@ -71,7 +70,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. @@ -79,11 +78,10 @@ Therefore, to make the test comparable, both AKS and Constellation on Azure were Constellation on Azure and AKS used an MTU of 1500. Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. - The difference in network bandwidth can largely be attributed to two factors. -* Constellation's [network encryption](../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. -* [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. +- Constellation's [network encryption](../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. ##### Pod-to-Pod @@ -134,6 +132,7 @@ The results for "Pod-to-Pod" on GCP are as follows: In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + #### Storage I/O Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). @@ -143,21 +142,25 @@ Similarly, upon a PVC request, Constellation will provision a PV via a default s For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + - 6400 (20000 burst) IOPS - 144 MB/s (600 MB/s burst) throughput However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + - 500 (600 burst) IOPS - 60 MB/s (150 MB/s burst) throughput For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + - 3,000 read IOPS - 15,000 write IOPS - 240 MB/s read throughput - 240 MB/s write throughput However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + - 2400 read IOPS - 2400 write IOPS - 112 MB/s read throughput @@ -178,8 +181,7 @@ The following `fio` settings were used: - IOPS: 4 KB blocks and 128 iodepth - Bandwidth: 1024 KB blocks and 128 iodepth -For more details, see the [`fio` test configuration](../../../../.github/actions/e2e_benchmark/fio.ini). - +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). The results for IOPS on Azure are as follows: diff --git a/docs/versioned_docs/version-2.6/overview/product.md b/docs/versioned_docs/version-2.6/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.6/overview/product.md +++ b/docs/versioned_docs/version-2.6/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.6/workflows/config.md b/docs/versioned_docs/version-2.6/workflows/config.md index 5da01beeb..56979ee13 100644 --- a/docs/versioned_docs/version-2.6/workflows/config.md +++ b/docs/versioned_docs/version-2.6/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,29 +14,29 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. @@ -47,25 +47,25 @@ You can also automatically generate a configuration file by adding the `--genera ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all nitroTPM-enabled machines with a minimum of 4 vCPUs (`xlarge` or larger). Refer to the [list of nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html) or run `constellation config instance-types` to get the list of all supported options. - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. @@ -79,8 +79,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you haven't generated a configuration file yet, you can do so by adding the `--generate-config` flag to the command. This creates a configuration file and populates it with the created IAM values. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session. @@ -103,23 +103,23 @@ Paste the output into the corresponding fields of the `constellation-conf.yaml` Since `clientSecretValue` is a sensitive value, you can leave it empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session. ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session. @@ -141,16 +141,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -194,19 +194,19 @@ The following describes the configuration fields and how you obtain the required Since this is a sensitive value, alternatively you can leave `clientSecretValue` empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -220,9 +220,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -251,9 +251,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.6/workflows/create.md b/docs/versioned_docs/version-2.6/workflows/create.md index d527618b0..81ed55582 100644 --- a/docs/versioned_docs/version-2.6/workflows/create.md +++ b/docs/versioned_docs/version-2.6/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + Choose the initial size of your cluster. The following command creates a cluster with one control-plane and two worker nodes: @@ -40,8 +40,8 @@ For details on the flags, consult the command help via `constellation create -h` *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -80,8 +80,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step diff --git a/docs/versioned_docs/version-2.6/workflows/recovery.md b/docs/versioned_docs/version-2.6/workflows/recovery.md index c26fb32eb..35596b8c9 100644 --- a/docs/versioned_docs/version-2.6/workflows/recovery.md +++ b/docs/versioned_docs/version-2.6/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.6/workflows/sbom.md b/docs/versioned_docs/version-2.6/workflows/sbom.md index 44b347a55..92550c182 100644 --- a/docs/versioned_docs/version-2.6/workflows/sbom.md +++ b/docs/versioned_docs/version-2.6/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -19,7 +19,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -40,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.6/workflows/scale.md b/docs/versioned_docs/version-2.6/workflows/scale.md index 3b7c0d479..bce045c66 100644 --- a/docs/versioned_docs/version-2.6/workflows/scale.md +++ b/docs/versioned_docs/version-2.6/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + :::caution @@ -72,8 +72,8 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + ## Control-plane node scaling @@ -81,24 +81,24 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + :::caution @@ -106,7 +106,7 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.6/workflows/storage.md b/docs/versioned_docs/version-2.6/workflows/storage.md index d0e5b188f..be9998676 100644 --- a/docs/versioned_docs/version-2.6/workflows/storage.md +++ b/docs/versioned_docs/version-2.6/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -57,8 +57,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -86,8 +86,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -115,8 +115,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + :::caution @@ -126,8 +126,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) @@ -186,8 +186,8 @@ The default storage class is responsible for all persistent volume claims that d Constellation creates a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -233,8 +233,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -280,8 +280,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -291,5 +291,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.6/workflows/terminate.md b/docs/versioned_docs/version-2.6/workflows/terminate.md index 647eadb42..f33489ca5 100644 --- a/docs/versioned_docs/version-2.6/workflows/terminate.md +++ b/docs/versioned_docs/version-2.6/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.6/workflows/troubleshooting.md b/docs/versioned_docs/version-2.6/workflows/troubleshooting.md index 801bb995a..6bdf1d75c 100644 --- a/docs/versioned_docs/version-2.6/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.6/workflows/troubleshooting.md @@ -10,6 +10,7 @@ If something doesn't work, check out the [known issues](https://github.com/edgel ## Azure: Resource Providers can't be registered On Azure, you may receive the following error when running `create` or `terminate` with limited IAM permissions: + ```shell-session Error: Error ensuring Resource Providers are registered. @@ -26,11 +27,13 @@ To continue, please ensure that the [required resource providers](../getting-sta Afterward, set `ARM_SKIP_PROVIDER_REGISTRATION=true` as an environment variable and either run `create` or `terminate` again. For example: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation create --control-plane-nodes 1 --worker-nodes 2 -y ``` Or alternatively, for `terminate`: + ```bash ARM_SKIP_PROVIDER_REGISTRATION=true constellation terminate ``` @@ -41,8 +44,8 @@ To provide information during early stages of the node's boot process, Constella You can view these information in the follow places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -52,8 +55,8 @@ You can view these information in the follow places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -68,16 +71,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ## Connect to nodes diff --git a/docs/versioned_docs/version-2.6/workflows/trusted-launch.md b/docs/versioned_docs/version-2.6/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.6/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.6/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.6/workflows/verify-cli.md b/docs/versioned_docs/version-2.6/workflows/verify-cli.md index 1280c51b0..aa2df4be4 100644 --- a/docs/versioned_docs/version-2.6/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.6/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.7/architecture/attestation.md b/docs/versioned_docs/version-2.7/architecture/attestation.md index 0c7a1487b..20f9909fd 100644 --- a/docs/versioned_docs/version-2.7/architecture/attestation.md +++ b/docs/versioned_docs/version-2.7/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,8 +217,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.7/architecture/keys.md b/docs/versioned_docs/version-2.7/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.7/architecture/keys.md +++ b/docs/versioned_docs/version-2.7/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.7/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.7/getting-started/first-steps-local.md index 707074bb9..81d8e141d 100644 --- a/docs/versioned_docs/version-2.7/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.7/getting-started/first-steps-local.md @@ -30,8 +30,8 @@ Both options use virtualization to create a local cluster with control-plane nod ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -59,8 +59,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -138,8 +138,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -192,8 +192,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -204,8 +204,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -233,8 +233,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.7/getting-started/first-steps.md b/docs/versioned_docs/version-2.7/getting-started/first-steps.md index 1569afc14..c1c3a3fe0 100644 --- a/docs/versioned_docs/version-2.7/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.7/getting-started/first-steps.md @@ -17,9 +17,9 @@ If you encounter any problem with the following steps, make sure to use the [lat First, you need to create a [configuration file](../workflows/config.md) and an [IAM configuration](../workflows/config.md#creating-an-iam-configuration). The easiest way to do this is the following CLI command: - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --generate-config @@ -33,21 +33,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `northeurope` * `westeurope` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --generate-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --generate-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=eu-central-1a --prefix=constellTest --generate-config @@ -66,8 +66,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.7/getting-started/install.md b/docs/versioned_docs/version-2.7/getting-started/install.md index 9ba727d81..ac0ef3da8 100644 --- a/docs/versioned_docs/version-2.7/getting-started/install.md +++ b/docs/versioned_docs/version-2.7/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,38 +105,41 @@ If you don't have a cloud subscription, you can try [MiniConstellation](first-st ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` The built-in `Contributor` role is a superset of these permissions. @@ -147,91 +147,91 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -366,8 +366,8 @@ The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -377,8 +377,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -394,8 +394,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -418,8 +418,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -435,10 +435,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.7/overview/clouds.md b/docs/versioned_docs/version-2.7/overview/clouds.md index c95b3508a..30995a012 100644 --- a/docs/versioned_docs/version-2.7/overview/clouds.md +++ b/docs/versioned_docs/version-2.7/overview/clouds.md @@ -24,24 +24,23 @@ The following table summarizes the state of features for different infrastructur With its [CVM offering](https://docs.microsoft.com/en-us/azure/confidential-computing/confidential-vm-overview), Azure provides the best foundations for Constellation. Regarding (3), Azure provides direct access to remote-attestation statements. However, regarding (4), the standard CVMs still include closed-source firmware running in VM Privilege Level (VMPL) 0. This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. Intel and Google have [collaborated](https://cloud.google.com/blog/products/identity-security/rsa-google-intel-confidential-computing-more-secure) to enhance the security of TDX, and have recently [revealed](https://venturebeat.com/security/intel-launches-confidential-computing-solution-for-virtual-machines/) their plans to make TDX compatible with Google Cloud. ## Amazon Web Services (AWS) + Amazon EC2 [supports AMD SEV-SNP](https://aws.amazon.com/de/about-aws/whats-new/2023/04/amazon-ec2-amd-sev-snp/). Regarding (3), AWS provides direct access to remote-attestation statements. However, attestation is partially based on the [NitroTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) for [measured boot](../architecture/attestation.md#measured-boot), which is a vTPM managed by the Nitro hypervisor. Hence, the hypervisor is currently part of Constellation's TCB. \* Regarding (4), the CVMs include initial firmware inside the CVM based on [OVMF](https://github.com/tianocore/tianocore.github.io/wiki/OVMF). Once this firmware will be reproducible and therefore verifiable, (4) switches from *No* to *Yes*. - - ## OpenStack OpenStack is an open-source cloud and infrastructure management software. It's used by many smaller CSPs and datacenters. In the latest *Yoga* version, OpenStack has basic support for CVMs. However, much depends on the employed kernel and hypervisor. Features (2)--(4) are likely to be a *Yes* with Linux kernel version 6.2. Thus, going forward, OpenStack on corresponding AMD or Intel hardware will be a viable underpinning for Constellation. diff --git a/docs/versioned_docs/version-2.7/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.7/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.7/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.7/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.7/overview/performance.md b/docs/versioned_docs/version-2.7/overview/performance.md index 54f31019a..aef594c46 100644 --- a/docs/versioned_docs/version-2.7/overview/performance.md +++ b/docs/versioned_docs/version-2.7/overview/performance.md @@ -63,7 +63,6 @@ The following infrastructure configurations was used: - CVM: `false` - Zone: `europe-west3-b` - ### Results #### Network @@ -71,7 +70,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. @@ -79,11 +78,10 @@ Therefore, to make the test comparable, both AKS and Constellation on Azure were Constellation on Azure and AKS used an MTU of 1500. Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. - The difference in network bandwidth can largely be attributed to two factors. -* Constellation's [network encryption](../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. -* [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. +- Constellation's [network encryption](../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. ##### Pod-to-Pod @@ -134,6 +132,7 @@ The results for "Pod-to-Pod" on GCP are as follows: In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + #### Storage I/O Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). @@ -143,21 +142,25 @@ Similarly, upon a PVC request, Constellation will provision a PV via a default s For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + - 6400 (20000 burst) IOPS - 144 MB/s (600 MB/s burst) throughput However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + - 500 (600 burst) IOPS - 60 MB/s (150 MB/s burst) throughput For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + - 3,000 read IOPS - 15,000 write IOPS - 240 MB/s read throughput - 240 MB/s write throughput However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + - 2400 read IOPS - 2400 write IOPS - 112 MB/s read throughput @@ -178,8 +181,7 @@ The following `fio` settings were used: - IOPS: 4 KB blocks and 128 iodepth - Bandwidth: 1024 KB blocks and 128 iodepth -For more details, see the [`fio` test configuration](../../../../.github/actions/e2e_benchmark/fio.ini). - +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). The results for IOPS on Azure are as follows: diff --git a/docs/versioned_docs/version-2.7/overview/product.md b/docs/versioned_docs/version-2.7/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.7/overview/product.md +++ b/docs/versioned_docs/version-2.7/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.7/workflows/config.md b/docs/versioned_docs/version-2.7/workflows/config.md index dd86a34a2..032b22943 100644 --- a/docs/versioned_docs/version-2.7/workflows/config.md +++ b/docs/versioned_docs/version-2.7/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,29 +14,29 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. @@ -47,25 +47,25 @@ You can also automatically generate a configuration file by adding the `--genera ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all nitroTPM-enabled machines with a minimum of 4 vCPUs (`xlarge` or larger). Refer to the [list of nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html) or run `constellation config instance-types` to get the list of all supported options. - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. @@ -79,8 +79,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you haven't generated a configuration file yet, you can do so by adding the `--generate-config` flag to the command. This creates a configuration file and populates it with the created IAM values. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -103,23 +103,23 @@ Paste the output into the corresponding fields of the `constellation-conf.yaml` Since `clientSecretValue` is a sensitive value, you can leave it empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -143,16 +143,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -196,19 +196,19 @@ The following describes the configuration fields and how you obtain the required Since this is a sensitive value, alternatively you can leave `clientSecretValue` empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -222,9 +222,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -255,9 +255,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.7/workflows/create.md b/docs/versioned_docs/version-2.7/workflows/create.md index aff59bb6a..5c4dd2948 100644 --- a/docs/versioned_docs/version-2.7/workflows/create.md +++ b/docs/versioned_docs/version-2.7/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + Choose the initial size of your cluster. The following command creates a cluster with one control-plane and two worker nodes: @@ -40,8 +40,8 @@ For details on the flags, consult the command help via `constellation create -h` *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -80,8 +80,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step diff --git a/docs/versioned_docs/version-2.7/workflows/recovery.md b/docs/versioned_docs/version-2.7/workflows/recovery.md index c26fb32eb..35596b8c9 100644 --- a/docs/versioned_docs/version-2.7/workflows/recovery.md +++ b/docs/versioned_docs/version-2.7/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.7/workflows/sbom.md b/docs/versioned_docs/version-2.7/workflows/sbom.md index 44b347a55..92550c182 100644 --- a/docs/versioned_docs/version-2.7/workflows/sbom.md +++ b/docs/versioned_docs/version-2.7/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -19,7 +19,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -40,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.7/workflows/scale.md b/docs/versioned_docs/version-2.7/workflows/scale.md index 3b7c0d479..bce045c66 100644 --- a/docs/versioned_docs/version-2.7/workflows/scale.md +++ b/docs/versioned_docs/version-2.7/workflows/scale.md @@ -48,23 +48,23 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + :::caution @@ -72,8 +72,8 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + ## Control-plane node scaling @@ -81,24 +81,24 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + :::caution @@ -106,7 +106,7 @@ Scaling isn't yet implemented for AWS. If you require this feature, [let us know ::: - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.7/workflows/storage.md b/docs/versioned_docs/version-2.7/workflows/storage.md index d0e5b188f..be9998676 100644 --- a/docs/versioned_docs/version-2.7/workflows/storage.md +++ b/docs/versioned_docs/version-2.7/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -57,8 +57,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -86,8 +86,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -115,8 +115,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + :::caution @@ -126,8 +126,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) @@ -186,8 +186,8 @@ The default storage class is responsible for all persistent volume claims that d Constellation creates a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -233,8 +233,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -280,8 +280,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -291,5 +291,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.7/workflows/terminate.md b/docs/versioned_docs/version-2.7/workflows/terminate.md index 647eadb42..f33489ca5 100644 --- a/docs/versioned_docs/version-2.7/workflows/terminate.md +++ b/docs/versioned_docs/version-2.7/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.7/workflows/troubleshooting.md b/docs/versioned_docs/version-2.7/workflows/troubleshooting.md index 2ddf3335d..cd095be28 100644 --- a/docs/versioned_docs/version-2.7/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.7/workflows/troubleshooting.md @@ -75,8 +75,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -86,8 +86,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -102,16 +102,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.7/workflows/trusted-launch.md b/docs/versioned_docs/version-2.7/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.7/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.7/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.7/workflows/verify-cli.md b/docs/versioned_docs/version-2.7/workflows/verify-cli.md index 1280c51b0..aa2df4be4 100644 --- a/docs/versioned_docs/version-2.7/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.7/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.8/architecture/attestation.md b/docs/versioned_docs/version-2.8/architecture/attestation.md index 07ac3aa72..592063193 100644 --- a/docs/versioned_docs/version-2.8/architecture/attestation.md +++ b/docs/versioned_docs/version-2.8/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,16 +217,16 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,18 +248,18 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + There is no additional configuration available for GCP. - - + + There is no additional configuration available for AWS. - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.8/architecture/keys.md b/docs/versioned_docs/version-2.8/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.8/architecture/keys.md +++ b/docs/versioned_docs/version-2.8/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.8/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.8/getting-started/first-steps-local.md index 707074bb9..81d8e141d 100644 --- a/docs/versioned_docs/version-2.8/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.8/getting-started/first-steps-local.md @@ -30,8 +30,8 @@ Both options use virtualization to create a local cluster with control-plane nod ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -59,8 +59,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -138,8 +138,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -192,8 +192,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -204,8 +204,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -233,8 +233,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.8/getting-started/first-steps.md b/docs/versioned_docs/version-2.8/getting-started/first-steps.md index 4449e6d37..8d5cc5cbb 100644 --- a/docs/versioned_docs/version-2.8/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.8/getting-started/first-steps.md @@ -17,9 +17,9 @@ If you encounter any problem with the following steps, make sure to use the [lat First, you need to create a [configuration file](../workflows/config.md) and an [IAM configuration](../workflows/config.md#creating-an-iam-configuration). The easiest way to do this is the following CLI command: - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --generate-config @@ -34,21 +34,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `westeurope` * `southeastasia` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --generate-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --generate-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also creates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=eu-central-1a --prefix=constellTest --generate-config @@ -67,8 +67,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.8/getting-started/install.md b/docs/versioned_docs/version-2.8/getting-started/install.md index 37940d0a2..2aa274b98 100644 --- a/docs/versioned_docs/version-2.8/getting-started/install.md +++ b/docs/versioned_docs/version-2.8/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,39 +105,42 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` -* `Microsoft.Network/natGateways/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` +- `Microsoft.Network/natGateways/*` The built-in `Contributor` role is a superset of these permissions. @@ -148,91 +148,91 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -367,8 +367,8 @@ The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -378,8 +378,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -395,8 +395,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -419,8 +419,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -436,10 +436,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.8/overview/clouds.md b/docs/versioned_docs/version-2.8/overview/clouds.md index 3ccbb0d6d..dfc3d5307 100644 --- a/docs/versioned_docs/version-2.8/overview/clouds.md +++ b/docs/versioned_docs/version-2.8/overview/clouds.md @@ -31,11 +31,11 @@ This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. diff --git a/docs/versioned_docs/version-2.8/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.8/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.8/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.8/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.8/overview/performance.md b/docs/versioned_docs/version-2.8/overview/performance.md index 54f31019a..aef594c46 100644 --- a/docs/versioned_docs/version-2.8/overview/performance.md +++ b/docs/versioned_docs/version-2.8/overview/performance.md @@ -63,7 +63,6 @@ The following infrastructure configurations was used: - CVM: `false` - Zone: `europe-west3-b` - ### Results #### Network @@ -71,7 +70,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. @@ -79,11 +78,10 @@ Therefore, to make the test comparable, both AKS and Constellation on Azure were Constellation on Azure and AKS used an MTU of 1500. Constellation on GCP used an MTU of 8896. GKE used an MTU of 1450. - The difference in network bandwidth can largely be attributed to two factors. -* Constellation's [network encryption](../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. -* [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. +- Constellation's [network encryption](../architecture/networking.md) via Cilium and WireGuard, which protects data in-transit. +- [AMD SEV using SWIOTLB bounce buffers](https://lore.kernel.org/all/20200204193500.GA15564@ashkalra_ubuntu_server/T/) for all DMA including network I/O. ##### Pod-to-Pod @@ -134,6 +132,7 @@ The results for "Pod-to-Pod" on GCP are as follows: In our recent comparison of Constellation on GCP with GKE, Constellation has 58% less TCP bandwidth. However, UDP bandwidth was slightly better with Constellation, thanks to its higher MTU. Similarly, when comparing Constellation on Azure with AKS using CVMs, Constellation achieved approximately 10% less TCP and 40% less UDP bandwidth. + #### Storage I/O Azure and GCP offer persistent storage for their Kubernetes services AKS and GKE via the Container Storage Interface (CSI). CSI storage in Kubernetes is available via `PersistentVolumes` (PV) and consumed via `PersistentVolumeClaims` (PVC). @@ -143,21 +142,25 @@ Similarly, upon a PVC request, Constellation will provision a PV via a default s For Constellation on Azure and AKS, the benchmark ran with Azure Disk storage [Standard SSD](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) of 400 GiB size. The [DC4as machine type](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) with four cores provides the following maximum performance: + - 6400 (20000 burst) IOPS - 144 MB/s (600 MB/s burst) throughput However, the performance is bound by the capabilities of the [512 GiB Standard SSD size](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#standard-ssds) (the size class of 400 GiB volumes): + - 500 (600 burst) IOPS - 60 MB/s (150 MB/s burst) throughput For Constellation on GCP and GKE, the benchmark ran with Compute Engine Persistent Disk Storage [pd-balanced](https://cloud.google.com/compute/docs/disks) of 400 GiB size. The N2D machine type with four cores and pd-balanced provides the following [maximum performance](https://cloud.google.com/compute/docs/disks/performance#n2d_vms): + - 3,000 read IOPS - 15,000 write IOPS - 240 MB/s read throughput - 240 MB/s write throughput However, the performance is bound by the capabilities of a [`Zonal balanced PD`](https://cloud.google.com/compute/docs/disks/performance#zonal-persistent-disks) with 400 GiB size: + - 2400 read IOPS - 2400 write IOPS - 112 MB/s read throughput @@ -178,8 +181,7 @@ The following `fio` settings were used: - IOPS: 4 KB blocks and 128 iodepth - Bandwidth: 1024 KB blocks and 128 iodepth -For more details, see the [`fio` test configuration](../../../../.github/actions/e2e_benchmark/fio.ini). - +For more details, see the [`fio` test configuration](https://github.com/edgelesssys/constellation/blob/main/.github/actions/e2e_benchmark/fio.ini). The results for IOPS on Azure are as follows: diff --git a/docs/versioned_docs/version-2.8/overview/product.md b/docs/versioned_docs/version-2.8/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.8/overview/product.md +++ b/docs/versioned_docs/version-2.8/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.8/workflows/config.md b/docs/versioned_docs/version-2.8/workflows/config.md index 260c4e6ec..f10ba14ec 100644 --- a/docs/versioned_docs/version-2.8/workflows/config.md +++ b/docs/versioned_docs/version-2.8/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,29 +14,29 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. @@ -47,25 +47,25 @@ You can also automatically generate a configuration file by adding the `--genera ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all nitroTPM-enabled machines with a minimum of 4 vCPUs (`xlarge` or larger). Refer to the [list of nitroTPM-enabled instance types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enable-nitrotpm-prerequisites.html) or run `constellation config instance-types` to get the list of all supported options. - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. @@ -79,8 +79,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you haven't generated a configuration file yet, you can do so by adding the `--generate-config` flag to the command. This creates a configuration file and populates it with the created IAM values. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -104,23 +104,23 @@ Paste the output into the corresponding fields of the `constellation-conf.yaml` Since `clientSecretValue` is a sensitive value, you can leave it empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -144,16 +144,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -198,19 +198,19 @@ The following describes the configuration fields and how you obtain the required Since this is a sensitive value, alternatively you can leave `clientSecretValue` empty in the configuration file and pass it via an environment variable instead. To this end, create the environment variable `CONSTELL_AZURE_CLIENT_SECRET_VALUE` and set it to the secret value. ::: - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -224,9 +224,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -257,9 +257,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.8/workflows/create.md b/docs/versioned_docs/version-2.8/workflows/create.md index aff59bb6a..5c4dd2948 100644 --- a/docs/versioned_docs/version-2.8/workflows/create.md +++ b/docs/versioned_docs/version-2.8/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + Choose the initial size of your cluster. The following command creates a cluster with one control-plane and two worker nodes: @@ -40,8 +40,8 @@ For details on the flags, consult the command help via `constellation create -h` *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -80,8 +80,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step diff --git a/docs/versioned_docs/version-2.8/workflows/recovery.md b/docs/versioned_docs/version-2.8/workflows/recovery.md index c26fb32eb..35596b8c9 100644 --- a/docs/versioned_docs/version-2.8/workflows/recovery.md +++ b/docs/versioned_docs/version-2.8/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.8/workflows/sbom.md b/docs/versioned_docs/version-2.8/workflows/sbom.md index c9dc0d5cc..6c1702dee 100644 --- a/docs/versioned_docs/version-2.8/workflows/sbom.md +++ b/docs/versioned_docs/version-2.8/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -19,7 +19,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -40,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.8/workflows/scale.md b/docs/versioned_docs/version-2.8/workflows/scale.md index 9531e90c9..46b048870 100644 --- a/docs/versioned_docs/version-2.8/workflows/scale.md +++ b/docs/versioned_docs/version-2.8/workflows/scale.md @@ -48,30 +48,30 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the worker ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + ## Control-plane node scaling @@ -79,30 +79,30 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.8/workflows/storage.md b/docs/versioned_docs/version-2.8/workflows/storage.md index d0e5b188f..be9998676 100644 --- a/docs/versioned_docs/version-2.8/workflows/storage.md +++ b/docs/versioned_docs/version-2.8/workflows/storage.md @@ -21,14 +21,14 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as ReadWriteOnce, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. @@ -36,8 +36,8 @@ This includes support for [volume snapshots](https://cloud.google.com/kubernetes You can use them to bring a volume back to a prior state or provision new volumes. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for information about the configuration. - - + + :::caution @@ -47,8 +47,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction) or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -57,8 +57,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -86,8 +86,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -115,8 +115,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + :::caution @@ -126,8 +126,8 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) @@ -186,8 +186,8 @@ The default storage class is responsible for all persistent volume claims that d Constellation creates a storage class with encryption enabled and sets this as the default class. In case you wish to change it, follow the steps below: - - + + 1. List the storage classes in your cluster: @@ -233,8 +233,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) azuredisk.csi.confidential.cloud Delete Immediate false 1d ``` - - + + 1. List the storage classes in your cluster: @@ -280,8 +280,8 @@ In case you wish to change it, follow the steps below: integrity-encrypted-rwo (default) gcp.csi.confidential.cloud Delete Immediate false 1d ``` - - + + :::caution @@ -291,5 +291,5 @@ You may use other (non-confidential) CSI drivers that are compatible with Kubern ::: - - + + diff --git a/docs/versioned_docs/version-2.8/workflows/terminate.md b/docs/versioned_docs/version-2.8/workflows/terminate.md index 647eadb42..f33489ca5 100644 --- a/docs/versioned_docs/version-2.8/workflows/terminate.md +++ b/docs/versioned_docs/version-2.8/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.8/workflows/troubleshooting.md b/docs/versioned_docs/version-2.8/workflows/troubleshooting.md index 2ddf3335d..cd095be28 100644 --- a/docs/versioned_docs/version-2.8/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.8/workflows/troubleshooting.md @@ -75,8 +75,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -86,8 +86,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -102,16 +102,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.8/workflows/trusted-launch.md b/docs/versioned_docs/version-2.8/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.8/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.8/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.8/workflows/verify-cli.md b/docs/versioned_docs/version-2.8/workflows/verify-cli.md index 1280c51b0..aa2df4be4 100644 --- a/docs/versioned_docs/version-2.8/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.8/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_docs/version-2.9/architecture/attestation.md b/docs/versioned_docs/version-2.9/architecture/attestation.md index 07ac3aa72..592063193 100644 --- a/docs/versioned_docs/version-2.9/architecture/attestation.md +++ b/docs/versioned_docs/version-2.9/architecture/attestation.md @@ -121,8 +121,8 @@ Constellation allows to specify in the config which measurements should be enfor Enforcing non-reproducible measurements controlled by the cloud provider means that changes in these values require manual updates to the cluster's config. By default, Constellation only enforces measurements that are stable values produced by the infrastructure or by Constellation directly. - - + + Constellation uses the [vTPM](https://docs.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm) feature of Azure CVMs for runtime measurements. This vTPM adheres to the [TPM 2.0](https://trustedcomputinggroup.org/resource/tpm-library-specification/) specification. @@ -152,8 +152,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://cloud.google.com/compute/confidential-vm/docs/about-cvm) feature of CVMs on GCP for runtime measurements. Note that this vTPM doesn't run inside the hardware-protected CVM context, but is emulated by the hypervisor. @@ -185,8 +185,8 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + Constellation uses the [vTPM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nitrotpm.html) (NitroTPM) feature of the [AWS Nitro System](http://aws.amazon.com/ec2/nitro/) on AWS for runtime measurements. @@ -217,16 +217,16 @@ The latter means that the value can be generated offline and compared to the one | 15 | ClusterID | Constellation Bootstrapper | Yes | | 16–23 | Unused | - | - | - - + + ### CVM verification To verify the integrity of the received attestation statement, a chain of trust from the CVM technology to the interface providing the statement has to be established. For verification of the CVM technology, Constellation may expose additional options in its config file. - - + + On Azure, AMD SEV-SNP is used to provide runtime encryption to the VMs. An SEV-SNP attestation report is used to establish trust in the vTPM running inside the VM. @@ -248,18 +248,18 @@ You may customize certain parameters for verification of the attestation stateme More explicitly, it controls the verification of the `IDKeyDigest` value in the SEV-SNP attestation report. You can provide a list of accepted key digests and specify a policy on how this list is compared against the reported `IDKeyDigest`. - - + + There is no additional configuration available for GCP. - - + + There is no additional configuration available for AWS. - - + + ## Cluster attestation diff --git a/docs/versioned_docs/version-2.9/architecture/keys.md b/docs/versioned_docs/version-2.9/architecture/keys.md index f2c8c3fba..553d9d4e2 100644 --- a/docs/versioned_docs/version-2.9/architecture/keys.md +++ b/docs/versioned_docs/version-2.9/architecture/keys.md @@ -105,7 +105,7 @@ Initially, it will support the following KMSs: * [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/#product-overview) * [KMIP-compatible KMS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=kmip) -Storing the keys in Cloud KMS of AWS, GCP, or Azure binds the key usage to the particular cloud identity access management (IAM). +Storing the keys in Cloud KMS of AWS, Azure, or GCP binds the key usage to the particular cloud identity access management (IAM). In the future, Constellation will support remote attestation-based access policies for Cloud KMS once available. Note that using a Cloud KMS limits the isolation and protection to the guarantees of the particular offering. diff --git a/docs/versioned_docs/version-2.9/getting-started/first-steps-local.md b/docs/versioned_docs/version-2.9/getting-started/first-steps-local.md index 8fcd71811..f9cfa5cd3 100644 --- a/docs/versioned_docs/version-2.9/getting-started/first-steps-local.md +++ b/docs/versioned_docs/version-2.9/getting-started/first-steps-local.md @@ -45,8 +45,8 @@ sudo iptables -P FORWARD ACCEPT ## Create a cluster - - + + With the `constellation mini` command, you can deploy and test Constellation locally. This mode is called MiniConstellation. Conceptually, MiniConstellation is similar to [MicroK8s](https://microk8s.io/), [K3s](https://k3s.io/), and [minikube](https://minikube.sigs.k8s.io/docs/). @@ -74,8 +74,8 @@ constellation mini up This will configure your current directory as the [workspace](../architecture/orchestration.md#workspaces) for this cluster. All `constellation` commands concerning this cluster need to be issued from this directory. - - + + With the QEMU provider, you can create a local Constellation cluster as if it were in the cloud. The provider uses [QEMU](https://www.qemu.org/) to create multiple VMs for the cluster nodes, which interact with each other. @@ -153,8 +153,8 @@ attaching persistent storage, or autoscaling aren't available. export KUBECONFIG="$PWD/constellation-admin.conf" ``` - - + + ## Connect to the cluster @@ -207,8 +207,8 @@ worker-0 Ready 32s v1.24.6 ## Terminate your cluster - - + + Once you are done, you can clean up the created resources using the following command: @@ -219,8 +219,8 @@ constellation mini down This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + Once you are done, you can clean up the created resources using the following command: @@ -248,8 +248,8 @@ Your Constellation cluster was terminated successfully. This will destroy your cluster and clean up your workspace. The VM image and cluster configuration file (`constellation-conf.yaml`) will be kept and may be reused to create new clusters. - - + + ## Troubleshooting diff --git a/docs/versioned_docs/version-2.9/getting-started/first-steps.md b/docs/versioned_docs/version-2.9/getting-started/first-steps.md index 0329c5776..6b0a06a06 100644 --- a/docs/versioned_docs/version-2.9/getting-started/first-steps.md +++ b/docs/versioned_docs/version-2.9/getting-started/first-steps.md @@ -15,39 +15,39 @@ If you encounter any problem with the following steps, make sure to use the [lat 1. Create the [configuration file](../workflows/config.md) for your cloud provider. - + - + ```bash constellation config generate azure ``` - + - + ```bash constellation config generate gcp ``` - + - + ```bash constellation config generate aws ``` - + - + 2. Create your [IAM configuration](../workflows/config.md#creating-an-iam-configuration). - + - + ```bash constellation iam create azure --region=westus --resourceGroup=constellTest --servicePrincipal=spTest --update-config @@ -62,21 +62,21 @@ If you encounter any problem with the following steps, make sure to use the [lat * `westeurope` * `southeastasia` - + - + ```bash - constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test --update-config + constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test --update-config ``` - This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. + This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. It also updates the configuration file `constellation-conf.yaml` in your current directory with the IAM values filled in. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `C2D` or `N2D`. - + - + ```bash constellation iam create aws --zone=us-east-2a --prefix=constellTest --update-config @@ -103,8 +103,8 @@ If you encounter any problem with the following steps, make sure to use the [lat You can find a list of all [regions in AWS's documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). - - + + :::tip To learn about all options you have for managing IAM resources and Constellation configuration, see the [Configuration workflow](../workflows/config.md). diff --git a/docs/versioned_docs/version-2.9/getting-started/install.md b/docs/versioned_docs/version-2.9/getting-started/install.md index 37940d0a2..2aa274b98 100644 --- a/docs/versioned_docs/version-2.9/getting-started/install.md +++ b/docs/versioned_docs/version-2.9/getting-started/install.md @@ -11,15 +11,15 @@ Make sure the following requirements are met: - Your machine is running Linux or macOS - You have admin rights on your machine - [kubectl](https://kubernetes.io/docs/tasks/tools/) is installed -- Your CSP is Microsoft Azure, Google Cloud Platform (GCP), or Amazon Web Services (AWS) +- Your CSP is Amazon Web Services (AWS), Microsoft Azure, or Google Cloud Platform (GCP) ## Install the Constellation CLI The CLI executable is available at [GitHub](https://github.com/edgelesssys/constellation/releases). Install it with the following commands: - - + + 1. Download the CLI: @@ -35,8 +35,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-amd64 /usr/local/bin/constellation ``` - - + + 1. Download the CLI: @@ -52,10 +52,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-linux-arm64 /usr/local/bin/constellation ``` + - - - + 1. Download the CLI: @@ -71,11 +70,9 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-arm64 /usr/local/bin/constellation ``` + - - - - + 1. Download the CLI: @@ -91,8 +88,8 @@ curl -LO https://github.com/edgelesssys/constellation/releases/latest/download/c sudo install constellation-darwin-amd64 /usr/local/bin/constellation ``` - - + + :::tip The CLI supports autocompletion for various shells. To set it up, run `constellation completion` and follow the given steps. @@ -108,39 +105,42 @@ If you don't have a cloud subscription, you can also set up a [local Constellati ### Required permissions - - + + The following [resource providers need to be registered](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types#register-resource-provider) in your subscription: -* `Microsoft.Attestation` \[2] -* `Microsoft.Compute` -* `Microsoft.Insights` -* `Microsoft.ManagedIdentity` -* `Microsoft.Network` + +- `Microsoft.Attestation` +- `Microsoft.Compute` +- `Microsoft.Insights` +- `Microsoft.ManagedIdentity` +- `Microsoft.Network` By default, Constellation tries to register these automatically if they haven't been registered before. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `*/register/action` \[1] -* `Microsoft.Authorization/roleAssignments/*` -* `Microsoft.Authorization/roleDefinitions/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Resources/subscriptions/resourcegroups/*` + +- `*/register/action` \[1] +- `Microsoft.Authorization/roleAssignments/*` +- `Microsoft.Authorization/roleDefinitions/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Resources/subscriptions/resourcegroups/*` The built-in `Owner` role is a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `Microsoft.Attestation/attestationProviders/*` \[2] -* `Microsoft.Compute/virtualMachineScaleSets/*` -* `Microsoft.Insights/components/*` -* `Microsoft.ManagedIdentity/userAssignedIdentities/*` -* `Microsoft.Network/loadBalancers/*` -* `Microsoft.Network/loadBalancers/backendAddressPools/*` -* `Microsoft.Network/networkSecurityGroups/*` -* `Microsoft.Network/publicIPAddresses/*` -* `Microsoft.Network/virtualNetworks/*` -* `Microsoft.Network/virtualNetworks/subnets/*` -* `Microsoft.Network/natGateways/*` + +- `Microsoft.Attestation/attestationProviders/*` +- `Microsoft.Compute/virtualMachineScaleSets/*` +- `Microsoft.Insights/components/*` +- `Microsoft.ManagedIdentity/userAssignedIdentities/*` +- `Microsoft.Network/loadBalancers/*` +- `Microsoft.Network/loadBalancers/backendAddressPools/*` +- `Microsoft.Network/networkSecurityGroups/*` +- `Microsoft.Network/publicIPAddresses/*` +- `Microsoft.Network/virtualNetworks/*` +- `Microsoft.Network/virtualNetworks/subnets/*` +- `Microsoft.Network/natGateways/*` The built-in `Contributor` role is a superset of these permissions. @@ -148,91 +148,91 @@ Follow Microsoft's guide on [understanding](https://learn.microsoft.com/en-us/az 1: You can omit `*/register/Action` if the resource providers mentioned above are already registered and the `ARM_SKIP_PROVIDER_REGISTRATION` environment variable is set to `true` when creating the IAM configuration. -2: You can omit `Microsoft.Attestation/attestationProviders/*` and the registration of `Microsoft.Attestation` if `EnforceIDKeyDigest` isn't set to `MAAFallback` in the [config file](../workflows/config.md#configure-your-cluster). - - - + + Create a new project for Constellation or use an existing one. Enable the [Compute Engine API](https://console.cloud.google.com/apis/library/compute.googleapis.com) on it. To [create the IAM configuration](../workflows/config.md#creating-an-iam-configuration) for Constellation, you need the following permissions: -* `iam.serviceAccountKeys.create` -* `iam.serviceAccountKeys.delete` -* `iam.serviceAccountKeys.get` -* `iam.serviceAccounts.create` -* `iam.serviceAccounts.delete` -* `iam.serviceAccounts.get` -* `resourcemanager.projects.getIamPolicy` -* `resourcemanager.projects.setIamPolicy` + +- `iam.serviceAccountKeys.create` +- `iam.serviceAccountKeys.delete` +- `iam.serviceAccountKeys.get` +- `iam.serviceAccounts.create` +- `iam.serviceAccounts.delete` +- `iam.serviceAccounts.get` +- `resourcemanager.projects.getIamPolicy` +- `resourcemanager.projects.setIamPolicy` Together, the built-in roles `roles/editor` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. To [create a Constellation cluster](../workflows/create.md#the-create-step), you need the following permissions: -* `compute.addresses.createInternal` -* `compute.addresses.deleteInternal` -* `compute.addresses.get` -* `compute.addresses.useInternal` -* `compute.backendServices.create` -* `compute.backendServices.delete` -* `compute.backendServices.get` -* `compute.backendServices.use` -* `compute.disks.create` -* `compute.firewalls.create` -* `compute.firewalls.delete` -* `compute.firewalls.get` -* `compute.globalAddresses.create` -* `compute.globalAddresses.delete` -* `compute.globalAddresses.get` -* `compute.globalAddresses.use` -* `compute.globalForwardingRules.create` -* `compute.globalForwardingRules.delete` -* `compute.globalForwardingRules.get` -* `compute.globalForwardingRules.setLabels` -* `compute.globalOperations.get` -* `compute.healthChecks.create` -* `compute.healthChecks.delete` -* `compute.healthChecks.get` -* `compute.healthChecks.useReadOnly` -* `compute.instanceGroupManagers.create` -* `compute.instanceGroupManagers.delete` -* `compute.instanceGroupManagers.get` -* `compute.instanceGroups.create` -* `compute.instanceGroups.delete` -* `compute.instanceGroups.get` -* `compute.instanceGroups.use` -* `compute.instances.create` -* `compute.instances.setLabels` -* `compute.instances.setMetadata` -* `compute.instances.setTags` -* `compute.instanceTemplates.create` -* `compute.instanceTemplates.delete` -* `compute.instanceTemplates.get` -* `compute.instanceTemplates.useReadOnly` -* `compute.networks.create` -* `compute.networks.delete` -* `compute.networks.get` -* `compute.networks.updatePolicy` -* `compute.routers.create` -* `compute.routers.delete` -* `compute.routers.get` -* `compute.routers.update` -* `compute.subnetworks.create` -* `compute.subnetworks.delete` -* `compute.subnetworks.get` -* `compute.subnetworks.use` -* `compute.targetTcpProxies.create` -* `compute.targetTcpProxies.delete` -* `compute.targetTcpProxies.get` -* `compute.targetTcpProxies.use` -* `iam.serviceAccounts.actAs` + +- `compute.addresses.createInternal` +- `compute.addresses.deleteInternal` +- `compute.addresses.get` +- `compute.addresses.useInternal` +- `compute.backendServices.create` +- `compute.backendServices.delete` +- `compute.backendServices.get` +- `compute.backendServices.use` +- `compute.disks.create` +- `compute.firewalls.create` +- `compute.firewalls.delete` +- `compute.firewalls.get` +- `compute.globalAddresses.create` +- `compute.globalAddresses.delete` +- `compute.globalAddresses.get` +- `compute.globalAddresses.use` +- `compute.globalForwardingRules.create` +- `compute.globalForwardingRules.delete` +- `compute.globalForwardingRules.get` +- `compute.globalForwardingRules.setLabels` +- `compute.globalOperations.get` +- `compute.healthChecks.create` +- `compute.healthChecks.delete` +- `compute.healthChecks.get` +- `compute.healthChecks.useReadOnly` +- `compute.instanceGroupManagers.create` +- `compute.instanceGroupManagers.delete` +- `compute.instanceGroupManagers.get` +- `compute.instanceGroups.create` +- `compute.instanceGroups.delete` +- `compute.instanceGroups.get` +- `compute.instanceGroups.use` +- `compute.instances.create` +- `compute.instances.setLabels` +- `compute.instances.setMetadata` +- `compute.instances.setTags` +- `compute.instanceTemplates.create` +- `compute.instanceTemplates.delete` +- `compute.instanceTemplates.get` +- `compute.instanceTemplates.useReadOnly` +- `compute.networks.create` +- `compute.networks.delete` +- `compute.networks.get` +- `compute.networks.updatePolicy` +- `compute.routers.create` +- `compute.routers.delete` +- `compute.routers.get` +- `compute.routers.update` +- `compute.subnetworks.create` +- `compute.subnetworks.delete` +- `compute.subnetworks.get` +- `compute.subnetworks.use` +- `compute.targetTcpProxies.create` +- `compute.targetTcpProxies.delete` +- `compute.targetTcpProxies.get` +- `compute.targetTcpProxies.use` +- `iam.serviceAccounts.actAs` Together, the built-in roles `roles/editor`, `roles/compute.instanceAdmin` and `roles/resourcemanager.projectIamAdmin` form a superset of these permissions. Follow Google's guide on [understanding](https://cloud.google.com/iam/docs/understanding-roles) and [assigning roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access). - - + + To set up a Constellation cluster, you need to perform two tasks that require permissions: create the infrastructure and create roles for cluster nodes. Both of these actions can be performed by different users, e.g., an administrator to create roles and a DevOps engineer to create the infrastructure. @@ -367,8 +367,8 @@ The built-in `PowerUserAccess` policy is a superset of these permissions. Follow Amazon's guide on [understanding](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) and [managing policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html). - - + + ### Authentication @@ -378,8 +378,8 @@ You need to authenticate with your CSP. The following lists the required steps f The steps for a *testing* environment are simpler. However, they may expose secrets to the CSP. If in doubt, follow the *production* steps. ::: - - + + **Testing** @@ -395,8 +395,8 @@ az login Other options are described in Azure's [authentication guide](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli). - - + + **Testing** @@ -419,8 +419,8 @@ Use one of the following options on a trusted machine: Follow [Google's guide](https://cloud.google.com/docs/authentication/production#manually) for setting up your credentials. - - + + **Testing** @@ -436,10 +436,9 @@ aws configure Options and first steps are described in the [AWS CLI documentation](https://docs.aws.amazon.com/cli/index.html). - + - - + ## Next steps diff --git a/docs/versioned_docs/version-2.9/overview/clouds.md b/docs/versioned_docs/version-2.9/overview/clouds.md index 3ccbb0d6d..dfc3d5307 100644 --- a/docs/versioned_docs/version-2.9/overview/clouds.md +++ b/docs/versioned_docs/version-2.9/overview/clouds.md @@ -31,11 +31,11 @@ This firmware is signed by Azure. The signature is reflected in the remote-attestation statements of CVMs. Thus, the Azure closed-source firmware becomes part of Constellation's trusted computing base (TCB). -\* Recently, Azure [announced](https://techcommunity.microsoft.com/t5/azure-confidential-computing/azure-confidential-vms-using-sev-snp-dcasv5-ecasv5-are-now/ba-p/3573747) the *limited preview* of CVMs with customizable firmware. With this CVM type, (4) switches from *No* to *Yes*. Constellation will support customizable firmware on Azure in the future. +\* Recently, [Azure announced the open source paravisor OpenHCL](https://techcommunity.microsoft.com/blog/windowsosplatform/openhcl-the-new-open-source-paravisor/4273172). It's the foundation for fully open source and verifiable CVM firmware. Once Azure provides their CVM firmware with reproducible builds based on OpenHCL, (4) switches from *No* to *Yes*. Constellation will support OpenHCL based firmware on Azure in the future. ## Google Cloud Platform (GCP) -The [CVMs Generally Available in GCP](https://cloud.google.com/compute/confidential-vm/docs/create-confidential-vm-instance) are based on AMD SEV but don't have SNP features enabled. +The [CVMs Generally Available in GCP](https://cloud.google.com/confidential-computing/confidential-vm/docs/confidential-vm-overview#amd_sev) are based on AMD SEV but don't have SNP features enabled. CVMs with SEV-SNP enabled are currently in [private preview](https://cloud.google.com/blog/products/identity-security/rsa-snp-vm-more-confidential). Regarding (3), with their SEV-SNP offering Google provides direct access to remote-attestation statements. However, regarding (4), the CVMs still include closed-source firmware. diff --git a/docs/versioned_docs/version-2.9/overview/confidential-kubernetes.md b/docs/versioned_docs/version-2.9/overview/confidential-kubernetes.md index 2b6c6ed17..1441c833a 100644 --- a/docs/versioned_docs/version-2.9/overview/confidential-kubernetes.md +++ b/docs/versioned_docs/version-2.9/overview/confidential-kubernetes.md @@ -23,9 +23,9 @@ With the above, Constellation wraps an entire cluster into one coherent and veri ![Confidential Kubernetes](../_media/concept-constellation.svg) -## Contrast: Managed Kubernetes with CVMs +## Comparison: Managed Kubernetes with CVMs -In contrast, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. +In comparison, managed Kubernetes with CVMs, as it's for example offered in [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) and [GKE](https://cloud.google.com/kubernetes-engine), only provides runtime encryption for certain worker nodes. Here, each worker node is a separate (and typically unverified) confidential context. This only provides limited security benefits as it only prevents direct access to a worker node's memory. The large majority of potential attacks through the infrastructure remain unaffected. This includes attacks through the control plane, access to external key management, and the corruption of worker node images. This leaves many problems unsolved. For instance, *Node A* has no means to verify if *Node B* is "good" and if it's OK to share data with it. Consequently, this approach leaves a large attack surface, as is depicted in the following. ![Concept: Managed Kubernetes plus CVMs](../_media/concept-managed.svg) diff --git a/docs/versioned_docs/version-2.9/overview/performance.md b/docs/versioned_docs/version-2.9/overview/performance.md index 9518ad538..aef594c46 100644 --- a/docs/versioned_docs/version-2.9/overview/performance.md +++ b/docs/versioned_docs/version-2.9/overview/performance.md @@ -70,7 +70,7 @@ The following infrastructure configurations was used: This section gives a thorough analysis of the network performance of Constellation, specifically focusing on measuring TCP and UDP bandwidth. The benchmark measured the bandwidth of pod-to-pod and pod-to-service connections between two different nodes using [`iperf`](https://iperf.fr/). -GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machineshttps://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). +GKE and Constellation on GCP had a maximum network bandwidth of [10 Gbps](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines). AKS with `Standard_D4as_v5` machines a maximum network bandwidth of [12.5 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series). The Confidential VM equivalent `Standard_DC4as_v5` currently has a network bandwidth of [1.25 Gbps](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series#dcasv5-series-products). Therefore, to make the test comparable, both AKS and Constellation on Azure were running with `Standard_DC4as_v5` machines and 1.25 Gbps bandwidth. diff --git a/docs/versioned_docs/version-2.9/overview/product.md b/docs/versioned_docs/version-2.9/overview/product.md index ba7181aa9..e42596fcc 100644 --- a/docs/versioned_docs/version-2.9/overview/product.md +++ b/docs/versioned_docs/version-2.9/overview/product.md @@ -6,6 +6,6 @@ From a security perspective, Constellation implements the [Confidential Kubernet From an operational perspective, Constellation provides the following key features: -* **Native support for different clouds**: Constellation works on Microsoft Azure, Google Cloud Platform (GCP), and Amazon Web Services (AWS). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). +* **Native support for different clouds**: Constellation works on Amazon Web Services (AWS), Microsoft Azure, and Google Cloud Platform (GCP). Support for OpenStack-based environments is coming with a future release. Constellation securely interfaces with the cloud infrastructure to provide [cluster autoscaling](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler), [dynamic persistent volumes](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/), and [service load balancing](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). * **High availability**: Constellation uses a [multi-master architecture](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/) with a [stacked etcd topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/#stacked-etcd-topology) to ensure high availability. * **Integrated Day-2 operations**: Constellation lets you securely [upgrade](../workflows/upgrade.md) your cluster to a new release. It also lets you securely [recover](../workflows/recovery.md) a failed cluster. Both with a single command. diff --git a/docs/versioned_docs/version-2.9/workflows/config.md b/docs/versioned_docs/version-2.9/workflows/config.md index f276f3f63..22a2821d8 100644 --- a/docs/versioned_docs/version-2.9/workflows/config.md +++ b/docs/versioned_docs/version-2.9/workflows/config.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -14,49 +14,49 @@ Before you can create your cluster, you need to configure the identity and acces You can generate a configuration file for your CSP by using the following CLI command: - - + + ```bash constellation config generate azure ``` - - + + ```bash constellation config generate gcp ``` - - + + ```bash constellation config generate aws ``` - - + + This creates the file `constellation-conf.yaml` in the current directory. ## Choosing a VM type Constellation supports the following VM types: - - + + By default, Constellation uses `Standard_DC4as_v5` CVMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. For CVMs, any VM type with a minimum of 4 vCPUs from the [DCasv5 & DCadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/dcasv5-dcadsv5-series) or [ECasv5 & ECadsv5](https://docs.microsoft.com/en-us/azure/virtual-machines/ecasv5-ecadsv5-series) families is supported. You can also run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `n2d-standard-4` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. Supported are all machines with a minimum of 4 vCPUs from the [C2D](https://cloud.google.com/compute/docs/compute-optimized-machines#c2d_machine_types) or [N2D](https://cloud.google.com/compute/docs/general-purpose-machines#n2d_machines) family. You can run `constellation config instance-types` to get the list of all supported options. - - + + By default, Constellation uses `m6a.xlarge` VMs (4 vCPUs, 16 GB RAM) to create your cluster. Optionally, you can switch to a different VM type by modifying **instanceType** in the configuration file. @@ -75,8 +75,8 @@ AWS is currently investigating the issue. SNP-based attestation will be enabled as soon as a fix is verified. ::: - - + + Fill the desired VM type into the **instanceType** field in the `constellation-conf.yml` file. @@ -90,8 +90,8 @@ See also Constellation's [Kubernetes support policy](../architecture/versions.md You can create an IAM configuration for your cluster automatically using the `constellation iam create` command. If you already have a Constellation configuration file, you can add the `--update-config` flag to the command. This writes the needed IAM fields into your configuration. Furthermore, the flag updates the zone/region of the configuration if it hasn't been set yet. - - + + You must be authenticated with the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -111,23 +111,23 @@ Note that CVMs are currently only supported in a few regions, check [Azure's pro Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [GCP CLI](https://cloud.google.com/sdk/gcloud) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). ```bash -constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west2-a --serviceAccountID=constell-test +constellation iam create gcp --projectID=yourproject-12345 --zone=europe-west3-a --serviceAccountID=constell-test ``` -This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west2-a` creating a new service account `constell-test`. +This command creates IAM configuration in the GCP project `yourproject-12345` on the GCP zone `europe-west3-a` creating a new service account `constell-test`. Note that only regions offering CVMs of the `C2D` or `N2D` series are supported. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available), which you can filter by machine type `N2D`. Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + + You must be authenticated with the [AWS CLI](https://aws.amazon.com/en/cli/) in the shell session with a user that has the [required permissions for IAM creation](../getting-started/install.md#set-up-cloud-credentials). @@ -151,16 +151,16 @@ You can find a list of all [regions in AWS's documentation](https://docs.aws.ama Paste the output into the corresponding fields of the `constellation-conf.yaml` file. - - + +
Alternatively, you can manually create the IAM configuration on your CSP. The following describes the configuration fields and how you obtain the required information or create the required resources. - - + + * **subscription**: The UUID of your Azure subscription, e.g., `8b8bd01f-efd9-4113-9bd1-c82137c32da7`. @@ -189,19 +189,19 @@ The following describes the configuration fields and how you obtain the required The user-assigned identity is used by instances of the cluster to access other cloud resources. For more information about managed identities refer to [Azure's documentation](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-manage-user-assigned-managed-identities). - + - + * **project**: The ID of your GCP project, e.g., `constellation-129857`. You can find it on the [welcome screen of your GCP project](https://console.cloud.google.com/welcome). For more information refer to [Google's documentation](https://support.google.com/googleapi/answer/7014113). -* **region**: The GCP region you want to deploy your cluster in, e.g., `us-west1`. +* **region**: The GCP region you want to deploy your cluster in, e.g., `us-central1`. You can find a [list of all regions in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). -* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-west1-a`. +* **zone**: The GCP zone you want to deploy your cluster in, e.g., `us-central1-a`. You can find a [list of all zones in Google's documentation](https://cloud.google.com/compute/docs/regions-zones#available). @@ -215,9 +215,9 @@ The following describes the configuration fields and how you obtain the required Afterward, create and download a new JSON key for this service account. Place the downloaded file in your Constellation workspace, and set the config parameter to the filename, e.g., `constellation-129857-15343dba46cb.json`. - + - + * **region**: The name of your chosen AWS data center region, e.g., `us-east-2`. @@ -248,9 +248,9 @@ The following describes the configuration fields and how you obtain the required Alternatively, you can create the AWS profile with a tool of your choice. Use the JSON policy in [main.tf](https://github.com/edgelesssys/constellation/tree/release/v2.2/hack/terraform/aws/iam/main.tf) in the resource `aws_iam_policy.worker_node_policy`. - + - +
Now that you've configured your CSP, you can [create your cluster](./create.md). diff --git a/docs/versioned_docs/version-2.9/workflows/create.md b/docs/versioned_docs/version-2.9/workflows/create.md index aff59bb6a..5c4dd2948 100644 --- a/docs/versioned_docs/version-2.9/workflows/create.md +++ b/docs/versioned_docs/version-2.9/workflows/create.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -26,8 +26,8 @@ Before you create the cluster, make sure to have a [valid configuration file](./ ### Create - - + + Choose the initial size of your cluster. The following command creates a cluster with one control-plane and two worker nodes: @@ -40,8 +40,8 @@ For details on the flags, consult the command help via `constellation create -h` *create* stores your cluster's state in a [`constellation-terraform`](../architecture/orchestration.md#cluster-creation-process) directory in your workspace. - - + + Terraform allows for an easier GitOps integration as well as meeting regulatory requirements. Since the Constellation CLI also uses Terraform under the hood, you can reuse the same Terraform files. @@ -80,8 +80,8 @@ CONSTELL_CSP=$(cat constellation-conf.yaml | yq ".provider | keys | .[0]") jq --null-input --arg cloudprovider "$CONSTELL_CSP" --arg ip "$CONSTELL_IP" --arg initsecret "$CONSTELL_INIT_SECRET" '{"cloudprovider":$cloudprovider,"ip":$ip,"initsecret":$initsecret}' > constellation-id.json ``` - - + + ## The *init* step diff --git a/docs/versioned_docs/version-2.9/workflows/recovery.md b/docs/versioned_docs/version-2.9/workflows/recovery.md index c26fb32eb..35596b8c9 100644 --- a/docs/versioned_docs/version-2.9/workflows/recovery.md +++ b/docs/versioned_docs/version-2.9/workflows/recovery.md @@ -16,8 +16,8 @@ You can check the health status of the nodes via the cloud service provider (CSP Constellation provides logging information on the boot process and status via [cloud logging](troubleshooting.md#cloud-logging). In the following, you'll find detailed descriptions for identifying clusters stuck in recovery for each CSP. - - + + In the Azure portal, find the cluster's resource group. Inside the resource group, open the control plane *Virtual machine scale set* `constellation-scale-set-controlplanes-`. @@ -51,8 +51,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, check that the control plane *Instance Group* has enough members in a *Ready* state. In the GCP Console, go to **Instance Groups** and check the group for the cluster's control plane `-control-plane-`. @@ -87,8 +87,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + First, open the AWS console to view all Auto Scaling Groups (ASGs) in the region of your cluster. Select the ASG of the control plane `--control-plane` and check that enough members are in a *Running* state. @@ -118,8 +118,8 @@ If this fails due to an unhealthy control plane, you will see log messages simil This means that you have to recover the node manually. - - + + ## Recover a cluster diff --git a/docs/versioned_docs/version-2.9/workflows/sbom.md b/docs/versioned_docs/version-2.9/workflows/sbom.md index c9dc0d5cc..6c1702dee 100644 --- a/docs/versioned_docs/version-2.9/workflows/sbom.md +++ b/docs/versioned_docs/version-2.9/workflows/sbom.md @@ -1,6 +1,6 @@ # Consume software bill of materials (SBOMs) - + --- @@ -19,7 +19,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). Make sure the key is available in a file named `cosign.pub` to execute the following examples. ::: @@ -40,7 +40,7 @@ cosign verify-blob --key cosign.pub --signature constellation.spdx.sbom.sig cons ### Container Images -SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/signing/other_types#sboms-software-bill-of-materials) and uploaded to the same registry. +SBOMs for container images are [attached to the image using Cosign](https://docs.sigstore.dev/cosign/signing/other_types/#sboms-software-bill-of-materials) and uploaded to the same registry. As a consumer, use cosign to download and verify the SBOM: diff --git a/docs/versioned_docs/version-2.9/workflows/scale.md b/docs/versioned_docs/version-2.9/workflows/scale.md index 06898ad0c..63b727c7d 100644 --- a/docs/versioned_docs/version-2.9/workflows/scale.md +++ b/docs/versioned_docs/version-2.9/workflows/scale.md @@ -51,30 +51,30 @@ kubectl -n kube-system get nodes Alternatively, you can manually scale your cluster up or down: - - + + 1. Find your Constellation resource group. 2. Select the `scale-set-workers`. 3. Go to **settings** and **scaling**. 4. Set the new **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **worker** instance group. 3. Set the new **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the worker ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + ## Control-plane node scaling @@ -82,30 +82,30 @@ Control-plane nodes can **only be scaled manually and only scaled up**! To increase the number of control-plane nodes, follow these steps: - + - + 1. Find your Constellation resource group. 2. Select the `scale-set-controlplanes`. 3. Go to **settings** and **scaling**. 4. Set the new (increased) **instance count** and **save**. - - + + 1. In Compute Engine go to [Instance Groups](https://console.cloud.google.com/compute/instanceGroups/). 2. **Edit** the **control-plane** instance group. 3. Set the new (increased) **number of instances** and **save**. - - + + 1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. 2. Click **Edit** 3. Set the new (increased) **Desired capacity** and **Update**. - - + + If you scale down the number of control-planes nodes, the removed nodes won't be able to exit the `etcd` cluster correctly. This will endanger the quorum that's required to run a stable Kubernetes control plane. diff --git a/docs/versioned_docs/version-2.9/workflows/storage.md b/docs/versioned_docs/version-2.9/workflows/storage.md index 9e3d96346..06fbc4de6 100644 --- a/docs/versioned_docs/version-2.9/workflows/storage.md +++ b/docs/versioned_docs/version-2.9/workflows/storage.md @@ -21,30 +21,30 @@ For more details see [encrypted persistent storage](../architecture/encrypted-st Constellation supports the following drivers, which offer node-level encryption and optional integrity protection. - - + + **Constellation CSI driver for Azure Disk**: Mount Azure [Disk Storage](https://azure.microsoft.com/en-us/services/storage/disks/#overview) into your Constellation cluster. See the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-azuredisk-csi-driver) for more information. Since Azure Disks are mounted as `ReadWriteOnce`, they're only available to a single pod. - - + + **Constellation CSI driver for GCP Persistent Disk**: Mount [Persistent Disk](https://cloud.google.com/persistent-disk) block storage into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver) for more information. - - + + **Constellation CSI driver for AWS Elastic Block Store** Mount [Elastic Block Store](https://aws.amazon.com/ebs/) storage volumes into your Constellation cluster. Follow the instructions on how to [install the Constellation CSI driver](#installation) or check out the [repository](https://github.com/edgelesssys/constellation-aws-ebs-csi-driver) for more information. - - + + Note that in case the options above aren't a suitable solution for you, Constellation is compatible with all other CSI-based storage options. For example, you can use [AWS EFS](https://docs.aws.amazon.com/en_en/eks/latest/userguide/efs-csi.html), [Azure Files](https://docs.microsoft.com/en-us/azure/storage/files/storage-files-introduction), or [GCP Filestore](https://cloud.google.com/filestore) with Constellation out of the box. Constellation is just not providing transparent encryption on the node level for these storage types yet. @@ -53,8 +53,8 @@ Note that in case the options above aren't a suitable solution for you, Constell The Constellation CLI automatically installs Constellation's CSI driver for the selected CSP in your cluster. If you don't need a CSI driver or wish to deploy your own, you can disable the automatic installation by setting `deployCSIDriver` to `false` in your Constellation config file. - - + + Azure comes with two storage classes by default. @@ -82,8 +82,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + GCP comes with two storage classes by default. @@ -111,8 +111,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + AWS comes with two storage classes by default. @@ -140,8 +140,8 @@ Note that volume expansion isn't supported for integrity-protected disks. ::: - - + + 1. Create a [persistent volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/docs/versioned_docs/version-2.9/workflows/terminate.md b/docs/versioned_docs/version-2.9/workflows/terminate.md index 647eadb42..f33489ca5 100644 --- a/docs/versioned_docs/version-2.9/workflows/terminate.md +++ b/docs/versioned_docs/version-2.9/workflows/terminate.md @@ -4,7 +4,7 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- @@ -16,8 +16,8 @@ All ephemeral storage and state of your cluster will be lost. Make sure any data ::: - - + + Terminate the cluster by running: ```bash @@ -40,8 +40,8 @@ resources manually. Just run the `terminate` command again afterward to continue ::: - - + + Terminate the cluster by running: ```bash @@ -56,5 +56,5 @@ rm constellation-id.json constellation-admin.conf Only the `constellation-mastersecret.json` and the configuration file remain. - - + + diff --git a/docs/versioned_docs/version-2.9/workflows/troubleshooting.md b/docs/versioned_docs/version-2.9/workflows/troubleshooting.md index 2ddf3335d..cd095be28 100644 --- a/docs/versioned_docs/version-2.9/workflows/troubleshooting.md +++ b/docs/versioned_docs/version-2.9/workflows/troubleshooting.md @@ -75,8 +75,8 @@ To provide information during early stages of a node's boot process, Constellati You can view this information in the following places: - - + + 1. In your Azure subscription find the Constellation resource group. 2. Inside the resource group find the Application Insights resource called `constellation-insights-*`. @@ -86,8 +86,8 @@ You can view this information in the following places: To **find the disk UUIDs** use the following query: `traces | where message contains "Disk UUID"` - - + + 1. Select the project that hosts Constellation. 2. Go to the `Compute Engine` service. @@ -102,16 +102,16 @@ Constellation uses the default bucket to store logs. Its [default retention peri ::: - - + + 1. Open [AWS CloudWatch](https://console.aws.amazon.com/cloudwatch/home) 2. Select [Log Groups](https://console.aws.amazon.com/cloudwatch/home#logsV2:log-groups) 3. Select the log group that matches the name of your cluster. 4. Select the log stream for control or worker type nodes. - - + + ### Node shell access diff --git a/docs/versioned_docs/version-2.9/workflows/trusted-launch.md b/docs/versioned_docs/version-2.9/workflows/trusted-launch.md index 13bd63ba6..11d0a096c 100644 --- a/docs/versioned_docs/version-2.9/workflows/trusted-launch.md +++ b/docs/versioned_docs/version-2.9/workflows/trusted-launch.md @@ -14,7 +14,7 @@ Constellation supports trusted launch VMs with instance types `Standard_D*_v4` a Azure currently doesn't support [community galleries for trusted launch VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/share-gallery-community). Thus, you need to manually import the Constellation node image into your cloud subscription. -The latest image is available at . Simply adjust the version number to download a newer version. +The latest image is available at `https://cdn.confidential.cloud/constellation/images/azure/trusted-launch/v2.2.0/constellation.img`. Simply adjust the version number to download a newer version. After you've downloaded the image, create a resource group `constellation-images` in your Azure subscription and import the image. You can use a script to do this: @@ -26,6 +26,7 @@ AZURE_IMAGE_VERSION=2.2.0 AZURE_RESOURCE_GROUP_NAME=constellation-images AZURE_I ``` The script creates the following resources: + 1. A new image gallery with the default name `constellation-import` 2. A new image definition with the default name `constellation` 3. The actual image with the provided version. In this case `2.2.0` diff --git a/docs/versioned_docs/version-2.9/workflows/verify-cli.md b/docs/versioned_docs/version-2.9/workflows/verify-cli.md index 1280c51b0..aa2df4be4 100644 --- a/docs/versioned_docs/version-2.9/workflows/verify-cli.md +++ b/docs/versioned_docs/version-2.9/workflows/verify-cli.md @@ -4,11 +4,11 @@ This recording presents the essence of this page. It's recommended to read it in full for the motivation and all details. ::: - + --- -Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/signing/quickstart), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at . +Edgeless Systems uses [sigstore](https://www.sigstore.dev/) and [SLSA](https://slsa.dev) to ensure supply-chain security for the Constellation CLI and node images ("artifacts"). sigstore consists of three components: [Cosign](https://docs.sigstore.dev/cosign/signing/overview/), [Rekor](https://docs.sigstore.dev/logging/overview), and Fulcio. Edgeless Systems uses Cosign to sign artifacts. All signatures are uploaded to the public Rekor transparency log, which resides at `https://rekor.sigstore.dev`. :::note The public key for Edgeless Systems' long-term code-signing key is: @@ -20,7 +20,7 @@ JmEe5iSLvG1SyQSAew7WdMKF6o9t8e2TFuCkzlOhhlws2OHWbiFZnFWCFw== -----END PUBLIC KEY----- ``` -The public key is also available for download at and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). +The public key is also available for download at [https://edgeless.systems/es.pub](https://edgeless.systems/es.pub) and in the Twitter profile [@EdgelessSystems](https://twitter.com/EdgelessSystems). ::: The Rekor transparency log is a public append-only ledger that verifies and records signatures and associated metadata. The Rekor transparency log enables everyone to observe the sequence of (software) signatures issued by Edgeless Systems and many other parties. The transparency log allows for the public identification of dubious or malicious signatures. @@ -33,7 +33,7 @@ You don't need to verify the Constellation node images. This is done automatical ## Verify the signature -First, [install the Cosign CLI](https://docs.sigstore.dev/system_config/installation). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: +First, [install the Cosign CLI](https://docs.sigstore.dev/cosign/system_config/installation/). Next, [download](https://github.com/edgelesssys/constellation/releases) and verify the signature that accompanies your CLI executable, for example: ```shell-session $ cosign verify-blob --key https://edgeless.systems/es.pub --signature constellation-linux-amd64.sig constellation-linux-amd64 diff --git a/docs/versioned_sidebars/version-2.10-sidebars.json b/docs/versioned_sidebars/version-2.10-sidebars.json index 02898994d..9d47f4b26 100644 --- a/docs/versioned_sidebars/version-2.10-sidebars.json +++ b/docs/versioned_sidebars/version-2.10-sidebars.json @@ -40,6 +40,11 @@ "id": "overview/performance/performance" }, "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, { "type": "doc", "label": "I/O benchmarks", diff --git a/docs/versioned_sidebars/version-2.11-sidebars.json b/docs/versioned_sidebars/version-2.11-sidebars.json index 17740bcca..8e0ad0ffb 100644 --- a/docs/versioned_sidebars/version-2.11-sidebars.json +++ b/docs/versioned_sidebars/version-2.11-sidebars.json @@ -40,6 +40,11 @@ "id": "overview/performance/performance" }, "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, { "type": "doc", "label": "I/O benchmarks", diff --git a/docs/versioned_sidebars/version-2.12-sidebars.json b/docs/versioned_sidebars/version-2.12-sidebars.json index 81aaba77d..e4c845754 100644 --- a/docs/versioned_sidebars/version-2.12-sidebars.json +++ b/docs/versioned_sidebars/version-2.12-sidebars.json @@ -40,6 +40,11 @@ "id": "overview/performance/performance" }, "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, { "type": "doc", "label": "I/O benchmarks", diff --git a/docs/versioned_sidebars/version-2.13-sidebars.json b/docs/versioned_sidebars/version-2.13-sidebars.json index 38caa4ac8..6317fc3f0 100644 --- a/docs/versioned_sidebars/version-2.13-sidebars.json +++ b/docs/versioned_sidebars/version-2.13-sidebars.json @@ -40,6 +40,11 @@ "id": "overview/performance/performance" }, "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, { "type": "doc", "label": "I/O benchmarks", diff --git a/docs/versioned_sidebars/version-2.14-sidebars.json b/docs/versioned_sidebars/version-2.14-sidebars.json new file mode 100644 index 000000000..ed97049b8 --- /dev/null +++ b/docs/versioned_sidebars/version-2.14-sidebars.json @@ -0,0 +1,294 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.15-sidebars.json b/docs/versioned_sidebars/version-2.15-sidebars.json new file mode 100644 index 000000000..09b5ec04e --- /dev/null +++ b/docs/versioned_sidebars/version-2.15-sidebars.json @@ -0,0 +1,299 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.16-sidebars.json b/docs/versioned_sidebars/version-2.16-sidebars.json new file mode 100644 index 000000000..09b5ec04e --- /dev/null +++ b/docs/versioned_sidebars/version-2.16-sidebars.json @@ -0,0 +1,299 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.17-sidebars.json b/docs/versioned_sidebars/version-2.17-sidebars.json new file mode 100644 index 000000000..09b5ec04e --- /dev/null +++ b/docs/versioned_sidebars/version-2.17-sidebars.json @@ -0,0 +1,299 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.18-sidebars.json b/docs/versioned_sidebars/version-2.18-sidebars.json new file mode 100644 index 000000000..09b5ec04e --- /dev/null +++ b/docs/versioned_sidebars/version-2.18-sidebars.json @@ -0,0 +1,299 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.19-sidebars.json b/docs/versioned_sidebars/version-2.19-sidebars.json new file mode 100644 index 000000000..09b5ec04e --- /dev/null +++ b/docs/versioned_sidebars/version-2.19-sidebars.json @@ -0,0 +1,299 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.20-sidebars.json b/docs/versioned_sidebars/version-2.20-sidebars.json new file mode 100644 index 000000000..c9937ab4f --- /dev/null +++ b/docs/versioned_sidebars/version-2.20-sidebars.json @@ -0,0 +1,304 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Reproduce release artifacts", + "id": "workflows/reproducible-builds" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.21-sidebars.json b/docs/versioned_sidebars/version-2.21-sidebars.json new file mode 100644 index 000000000..c9937ab4f --- /dev/null +++ b/docs/versioned_sidebars/version-2.21-sidebars.json @@ -0,0 +1,304 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Reproduce release artifacts", + "id": "workflows/reproducible-builds" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.22-sidebars.json b/docs/versioned_sidebars/version-2.22-sidebars.json new file mode 100644 index 000000000..c9937ab4f --- /dev/null +++ b/docs/versioned_sidebars/version-2.22-sidebars.json @@ -0,0 +1,304 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Reproduce release artifacts", + "id": "workflows/reproducible-builds" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versioned_sidebars/version-2.23-sidebars.json b/docs/versioned_sidebars/version-2.23-sidebars.json new file mode 100644 index 000000000..c9937ab4f --- /dev/null +++ b/docs/versioned_sidebars/version-2.23-sidebars.json @@ -0,0 +1,304 @@ +{ + "docs": [ + { + "type": "doc", + "label": "Introduction", + "id": "intro" + }, + { + "type": "category", + "label": "Basics", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Confidential Kubernetes", + "id": "overview/confidential-kubernetes" + }, + { + "type": "doc", + "label": "Security benefits", + "id": "overview/security-benefits" + }, + { + "type": "doc", + "label": "Product features", + "id": "overview/product" + }, + { + "type": "doc", + "label": "Feature status of clouds", + "id": "overview/clouds" + }, + { + "type": "category", + "label": "Performance", + "link": { + "type": "doc", + "id": "overview/performance/performance" + }, + "items": [ + { + "type": "doc", + "label": "Compute benchmarks", + "id": "overview/performance/compute" + }, + { + "type": "doc", + "label": "I/O benchmarks", + "id": "overview/performance/io" + }, + { + "type": "doc", + "label": "Application benchmarks", + "id": "overview/performance/application" + } + ] + }, + { + "type": "doc", + "label": "License", + "id": "overview/license" + } + ] + }, + { + "type": "category", + "label": "Getting started", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Installation", + "id": "getting-started/install" + }, + { + "type": "doc", + "label": "First steps (cloud)", + "id": "getting-started/first-steps" + }, + { + "type": "doc", + "label": "First steps (local)", + "id": "getting-started/first-steps-local" + }, + { + "type": "doc", + "label": "Cloud Marketplaces", + "id": "getting-started/marketplaces" + }, + { + "type": "category", + "label": "Examples", + "link": { + "type": "doc", + "id": "getting-started/examples" + }, + "items": [ + { + "type": "doc", + "label": "Emojivoto", + "id": "getting-started/examples/emojivoto" + }, + { + "type": "doc", + "label": "Online Boutique", + "id": "getting-started/examples/online-boutique" + }, + { + "type": "doc", + "label": "Horizontal Pod Autoscaling", + "id": "getting-started/examples/horizontal-scaling" + }, + { + "type": "doc", + "label": "Filestash with s3proxy", + "id": "getting-started/examples/filestash-s3proxy" + } + ] + } + ] + }, + { + "type": "category", + "label": "Workflows", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Verify the CLI", + "id": "workflows/verify-cli" + }, + { + "type": "doc", + "label": "Configure your cluster", + "id": "workflows/config" + }, + { + "type": "doc", + "label": "Create your cluster", + "id": "workflows/create" + }, + { + "type": "doc", + "label": "Scale your cluster", + "id": "workflows/scale" + }, + { + "type": "doc", + "label": "Upgrade your cluster", + "id": "workflows/upgrade" + }, + { + "type": "doc", + "label": "Expose a service", + "id": "workflows/lb" + }, + { + "type": "doc", + "label": "Install cert-manager", + "id": "workflows/cert-manager" + }, + { + "type": "doc", + "label": "Install s3proxy", + "id": "workflows/s3proxy" + }, + { + "type": "doc", + "label": "Terminate your cluster", + "id": "workflows/terminate" + }, + { + "type": "doc", + "label": "Recover your cluster", + "id": "workflows/recovery" + }, + { + "type": "doc", + "label": "Verify your cluster", + "id": "workflows/verify-cluster" + }, + { + "type": "doc", + "label": "Use persistent storage", + "id": "workflows/storage" + }, + { + "type": "doc", + "label": "Use the Terraform provider", + "id": "workflows/terraform-provider" + }, + { + "type": "doc", + "label": "Consume SBOMs", + "id": "workflows/sbom" + }, + { + "type": "doc", + "label": "Reproduce release artifacts", + "id": "workflows/reproducible-builds" + }, + { + "type": "doc", + "label": "Troubleshooting", + "id": "workflows/troubleshooting" + } + ] + }, + { + "type": "category", + "label": "Architecture", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "Overview", + "id": "architecture/overview" + }, + { + "type": "doc", + "label": "Cluster orchestration", + "id": "architecture/orchestration" + }, + { + "type": "doc", + "label": "Versions and support", + "id": "architecture/versions" + }, + { + "type": "doc", + "label": "Microservices", + "id": "architecture/microservices" + }, + { + "type": "doc", + "label": "Attestation", + "id": "architecture/attestation" + }, + { + "type": "doc", + "label": "Images", + "id": "architecture/images" + }, + { + "type": "doc", + "label": "Keys and cryptographic primitives", + "id": "architecture/keys" + }, + { + "type": "doc", + "label": "Encrypted persistent storage", + "id": "architecture/encrypted-storage" + }, + { + "type": "doc", + "label": "Networking", + "id": "architecture/networking" + }, + { + "type": "doc", + "label": "Observability", + "id": "architecture/observability" + } + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "generated-index" + }, + "items": [ + { + "type": "doc", + "label": "CLI", + "id": "reference/cli" + }, + { + "type": "doc", + "label": "Configuration migrations", + "id": "reference/migration" + }, + { + "type": "doc", + "label": "Terraform usage", + "id": "reference/terraform" + }, + { + "type": "doc", + "label": "SLSA adoption", + "id": "reference/slsa" + } + ] + } + ] +} diff --git a/docs/versions.json b/docs/versions.json index 7133c9849..8cc6150df 100644 --- a/docs/versions.json +++ b/docs/versions.json @@ -1,4 +1,14 @@ [ + "2.23", + "2.22", + "2.21", + "2.20", + "2.19", + "2.18", + "2.17", + "2.16", + "2.15", + "2.14", "2.13", "2.12", "2.11", diff --git a/e2e/e2e.go b/e2e/e2e.go index 43a0044e7..4c23c394c 100644 --- a/e2e/e2e.go +++ b/e2e/e2e.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // End-to-end tests which are executed from our GitHub action pipelines. diff --git a/e2e/internal/kubectl/kubectl.go b/e2e/internal/kubectl/kubectl.go index 2fb191b30..e44abd446 100644 --- a/e2e/internal/kubectl/kubectl.go +++ b/e2e/internal/kubectl/kubectl.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Provides functionality to easily interact with the K8s API, which can be used diff --git a/e2e/internal/lb/lb.go b/e2e/internal/lb/lb.go index 2cd01237b..b38202079 100644 --- a/e2e/internal/lb/lb.go +++ b/e2e/internal/lb/lb.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package lb tests that the cloud load balancer works as expected. diff --git a/e2e/internal/lb/lb_test.go b/e2e/internal/lb/lb_test.go index 94c8d2ff3..c8a3d2d16 100644 --- a/e2e/internal/lb/lb_test.go +++ b/e2e/internal/lb/lb_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // End-to-end tests for our cloud load balancer functionality. @@ -12,7 +12,6 @@ package lb import ( "bufio" "bytes" - "context" "fmt" "io" "net/http" @@ -70,7 +69,7 @@ func TestLoadBalancer(t *testing.T) { t.Log("Change port of service to 8044") svc.Spec.Ports[0].Port = newPort - svc, err = k.CoreV1().Services(namespaceName).Update(context.Background(), svc, metaV1.UpdateOptions{}) + svc, err = k.CoreV1().Services(namespaceName).Update(t.Context(), svc, metaV1.UpdateOptions{}) require.NoError(err) assert.Equal(newPort, svc.Spec.Ports[0].Port) @@ -93,7 +92,7 @@ func gatherDebugInfo(t *testing.T, k *kubernetes.Clientset) { t.Log("Gathering additional debug information.") - pods, err := k.CoreV1().Pods(namespaceName).List(context.Background(), metaV1.ListOptions{ + pods, err := k.CoreV1().Pods(namespaceName).List(t.Context(), metaV1.ListOptions{ LabelSelector: "app=whoami", }) if err != nil { @@ -106,7 +105,7 @@ func gatherDebugInfo(t *testing.T, k *kubernetes.Clientset) { req := k.CoreV1().Pods(namespaceName).GetLogs(pod.Name, &coreV1.PodLogOptions{ LimitBytes: func() *int64 { i := int64(1024 * 1024); return &i }(), }) - logs, err := req.Stream(context.Background()) + logs, err := req.Stream(t.Context()) if err != nil { t.Logf("fetching logs: %v", err) return @@ -155,7 +154,7 @@ func testEventuallyStatusOK(t *testing.T, url string) { require := require.New(t) assert.Eventually(func() bool { - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, http.NoBody) require.NoError(err) resp, err := http.DefaultClient.Do(req) @@ -183,7 +182,7 @@ func testEventuallyExternalIPAvailable(t *testing.T, k *kubernetes.Clientset) *c require.Eventually(t, func() bool { var err error - svc, err = k.CoreV1().Services(namespaceName).Get(context.Background(), serviceName, metaV1.GetOptions{}) + svc, err = k.CoreV1().Services(namespaceName).Get(t.Context(), serviceName, metaV1.GetOptions{}) if err != nil { t.Log("Getting service failed: ", err.Error()) return false @@ -212,7 +211,7 @@ func testEndpointAvailable(t *testing.T, url string, allHostnames []string, reqI assert := assert.New(t) require := require.New(t) - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, http.NoBody) require.NoError(err) resp, err := http.DefaultClient.Do(req) diff --git a/e2e/internal/upgrade/BUILD.bazel b/e2e/internal/upgrade/BUILD.bazel index 2d16064bd..b97119e5f 100644 --- a/e2e/internal/upgrade/BUILD.bazel +++ b/e2e/internal/upgrade/BUILD.bazel @@ -10,9 +10,19 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/e2e/internal/upgrade", visibility = ["//e2e:__subpackages__"], deps = [ + "//internal/api/attestationconfigapi", + "//internal/config", "//internal/constants", + "//internal/file", + "//internal/imagefetcher", "//internal/logger", "//internal/semver", + "//internal/versions", + "@com_github_spf13_afero//:afero", + "@com_github_stretchr_testify//require", + "@io_bazel_rules_go//go/runfiles", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_client_go//kubernetes", "@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/cli", ], @@ -35,16 +45,10 @@ go_test( tags = ["manual"], deps = [ "//e2e/internal/kubectl", - "//internal/api/attestationconfigapi", - "//internal/config", "//internal/constants", - "//internal/file", - "//internal/imagefetcher", - "//internal/semver", "//internal/versions", - "@com_github_spf13_afero//:afero", + "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_client_go//kubernetes", diff --git a/e2e/internal/upgrade/helm.go b/e2e/internal/upgrade/helm.go index b0fc498fd..23b55d327 100644 --- a/e2e/internal/upgrade/helm.go +++ b/e2e/internal/upgrade/helm.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package upgrade @@ -25,7 +25,7 @@ func servicesVersion(t *testing.T) (semver.Semver, error) { settings := cli.New() settings.KubeConfig = "constellation-admin.conf" actionConfig := &action.Configuration{} - if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, "secret", log.Infof); err != nil { + if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, "secret", log.Info); err != nil { return semver.Semver{}, fmt.Errorf("initializing config: %w", err) } diff --git a/e2e/internal/upgrade/upgrade.go b/e2e/internal/upgrade/upgrade.go index 6fa503655..09452bf13 100644 --- a/e2e/internal/upgrade/upgrade.go +++ b/e2e/internal/upgrade/upgrade.go @@ -1,7 +1,9 @@ +//go:build e2e + /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package upgrade tests that the CLI's apply command works as expected and @@ -17,3 +19,311 @@ SPDX-License-Identifier: AGPL-3.0-only // // - set or fetch measurements depending on target image package upgrade + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/imagefetcher" + "github.com/edgelesssys/constellation/v2/internal/semver" + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// tickDuration is the duration between two checks to see if the upgrade is successful. +var tickDuration = 10 * time.Second // small tick duration to speed up tests + +// VersionContainer contains the versions that the cluster should be upgraded to. +type VersionContainer struct { + ImageRef string + Kubernetes versions.ValidK8sVersion + Microservices semver.Semver +} + +// AssertUpgradeSuccessful tests that the upgrade to the target version is successful. +func AssertUpgradeSuccessful(t *testing.T, cli string, targetVersions VersionContainer, k *kubernetes.Clientset, wantControl, wantWorker int, timeout time.Duration) { + wg := queryStatusAsync(t, cli) + require.NotNil(t, k) + + testMicroservicesEventuallyHaveVersion(t, targetVersions.Microservices, timeout) + log.Println("Microservices are upgraded.") + + testNodesEventuallyHaveVersion(t, k, targetVersions, wantControl+wantWorker, timeout) + log.Println("Nodes are upgraded.") + wg.Wait() +} + +func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + // The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy. + // Nodes can upgrade in <5mins. + testStatusEventuallyWorks(t, cli, 20*time.Minute) + }() + + return &wg +} + +func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) { + require.Eventually(t, func() bool { + // Show versions set in cluster. + // The string after "Cluster status:" in the output might not be updated yet. + // This is only updated after the operator finishes one reconcile loop. + cmd := exec.CommandContext(t.Context(), cli, "status") + stdout, stderr, err := runCommandWithSeparateOutputs(cmd) + if err != nil { + log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr)) + return false + } + + log.Println(string(stdout)) + return true + }, timeout, tickDuration) +} + +func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) { + require.Eventually(t, func() bool { + version, err := servicesVersion(t) + if err != nil { + log.Printf("Unable to fetch microservice version: %v\n", err) + return false + } + + if version != wantMicroserviceVersion { + log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion) + return false + } + + return true + }, timeout, tickDuration) +} + +func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions VersionContainer, totalNodeCount int, timeout time.Duration) { + require.Eventually(t, func() bool { + nodes, err := k.CoreV1().Nodes().List(t.Context(), metaV1.ListOptions{}) + if err != nil { + log.Println(err) + return false + } + + // require is not printed in the logs, so we use fmt + tooSmallNodeCount := len(nodes.Items) < totalNodeCount + if tooSmallNodeCount { + log.Printf("expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items)) + return false + } + + allUpdated := true + log.Printf("Node status (%v):", time.Now()) + for _, node := range nodes.Items { + for key, value := range node.Annotations { + if targetVersions.ImageRef != "" { + if key == "constellation.edgeless.systems/node-image" { + if !strings.EqualFold(value, targetVersions.ImageRef) { + log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.ImageRef) + allUpdated = false + } + } + } + } + if targetVersions.Kubernetes != "" { + kubeletVersion := node.Status.NodeInfo.KubeletVersion + if kubeletVersion != string(targetVersions.Kubernetes) { + log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.Kubernetes) + allUpdated = false + } + } + } + return allUpdated + }, timeout, tickDuration) +} + +// runCommandWithSeparateOutputs runs the given command while separating buffers for +// stdout and stderr. +func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) { + stdout = []byte{} + stderr = []byte{} + + stdoutIn, err := cmd.StdoutPipe() + if err != nil { + err = fmt.Errorf("create stdout pipe: %w", err) + return + } + stderrIn, err := cmd.StderrPipe() + if err != nil { + err = fmt.Errorf("create stderr pipe: %w", err) + return + } + + err = cmd.Start() + if err != nil { + err = fmt.Errorf("start command: %w", err) + return + } + + continuouslyPrintOutput := func(r io.Reader, prefix string, wg *sync.WaitGroup) { + defer wg.Done() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + output := scanner.Text() + fmt.Printf("%s: %s\n", prefix, output) + switch prefix { + case "stdout": + stdout = append(stdout, output...) + case "stderr": + stderr = append(stderr, output...) + } + } + } + + wg := &sync.WaitGroup{} + wg.Add(2) + go continuouslyPrintOutput(stdoutIn, "stdout", wg) + go continuouslyPrintOutput(stderrIn, "stderr", wg) + + if err = cmd.Wait(); err != nil { + err = fmt.Errorf("wait for command to finish: %w", err) + } + wg.Wait() + + return stdout, stderr, err +} + +// Setup checks that the prerequisites for the test are met: +// - a workspace is set +// - a CLI path is set +// - the constellation-upgrade folder does not exist. +func Setup(workspace, cliPath string) error { + workingDir, err := workingDir(workspace) + if err != nil { + return fmt.Errorf("getting working directory: %w", err) + } + + if err := os.Chdir(workingDir); err != nil { + return fmt.Errorf("changing working directory: %w", err) + } + + if _, err := getCLIPath(cliPath); err != nil { + return fmt.Errorf("getting CLI path: %w", err) + } + return nil +} + +// workingDir returns the path to the workspace. +func workingDir(workspace string) (string, error) { + workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") + switch { + case workingDir != "": + return workingDir, nil + case workspace != "": + return workspace, nil + default: + return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") + } +} + +// WriteUpgradeConfig writes the target versions to the config file. +func WriteUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string, configPath string) VersionContainer { + fileHandler := file.NewHandler(afero.NewOsFs()) + attestationFetcher := attestationconfigapi.NewFetcher() + cfg, err := config.New(fileHandler, configPath, attestationFetcher, true) + var cfgErr *config.ValidationError + var longMsg string + if errors.As(err, &cfgErr) { + longMsg = cfgErr.LongMessage() + } + require.NoError(err, longMsg) + + imageFetcher := imagefetcher.New() + imageRef, err := imageFetcher.FetchReference( + context.Background(), + cfg.GetProvider(), + cfg.GetAttestationConfig().GetVariant(), + image, + cfg.GetRegion(), cfg.UseMarketplaceImage(), + ) + require.NoError(err) + + log.Printf("Setting image version: %s\n", image) + cfg.Image = image + + defaultConfig := config.Default() + var kubernetesVersion versions.ValidK8sVersion + if kubernetes == "" { + kubernetesVersion = defaultConfig.KubernetesVersion + } else { + kubernetesVersion = versions.ValidK8sVersion(kubernetes) // ignore validation because the config is only written to file + } + + var microserviceVersion semver.Semver + if microservices == "" { + microserviceVersion = defaultConfig.MicroserviceVersion + } else { + version, err := semver.New(microservices) + require.NoError(err) + microserviceVersion = version + } + + log.Printf("Setting K8s version: %s\n", kubernetesVersion) + cfg.KubernetesVersion = kubernetesVersion + log.Printf("Setting microservice version: %s\n", microserviceVersion) + cfg.MicroserviceVersion = microserviceVersion + + err = fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptOverwrite) + require.NoError(err) + + return VersionContainer{ImageRef: imageRef, Kubernetes: kubernetesVersion, Microservices: microserviceVersion} +} + +// getCLIPath returns the path to the CLI. +func getCLIPath(cliPathFlag string) (string, error) { + pathCLI := os.Getenv("PATH_CLI") + var relCLIPath string + switch { + case cliPathFlag != "": + relCLIPath = cliPathFlag + case pathCLI != "": + relCLIPath = pathCLI + default: + return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") + } + + // try to find the CLI in the working directory + // (e.g. when running via `go test` or when specifying a path manually) + workdir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("getting working directory: %w", err) + } + + absCLIPath := relCLIPath + if !filepath.IsAbs(relCLIPath) { + absCLIPath = filepath.Join(workdir, relCLIPath) + } + if _, err := os.Stat(absCLIPath); err == nil { + return absCLIPath, nil + } + + // fall back to runfiles (e.g. when running via bazel) + return runfiles.Rlocation(pathCLI) +} diff --git a/e2e/internal/upgrade/upgrade_test.go b/e2e/internal/upgrade/upgrade_test.go index bf9da20e2..6f1a7b517 100644 --- a/e2e/internal/upgrade/upgrade_test.go +++ b/e2e/internal/upgrade/upgrade_test.go @@ -3,37 +3,27 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package upgrade import ( - "bufio" "context" "errors" "flag" "fmt" - "io" "log" "os" "os/exec" - "path/filepath" "strings" - "sync" "testing" "time" - "github.com/bazelbuild/rules_go/go/runfiles" "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" - "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" - "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/file" - "github.com/edgelesssys/constellation/v2/internal/imagefetcher" - "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" - "github.com/spf13/afero" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" coreV1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,8 +42,8 @@ var ( // When executing the test as a bazel target the CLI path is supplied through an env variable that bazel sets. // When executing via `go test` extra care should be taken that the supplied CLI is built on the same commit as this test. cliPath = flag.String("cli", "", "Constellation CLI to run the tests.") - wantWorker = flag.Int("want-worker", 0, "Number of wanted worker nodes.") - wantControl = flag.Int("want-control", 0, "Number of wanted control nodes.") + wantWorker = flag.Int("want-worker", 1, "Number of wanted worker nodes.") + wantControl = flag.Int("want-control", 1, "Number of wanted control nodes.") timeout = flag.Duration("timeout", 3*time.Hour, "Timeout after which the cluster should have converged to the target version.") ) @@ -63,7 +53,7 @@ var ( func TestUpgrade(t *testing.T) { require := require.New(t) - err := setup() + err := Setup(*workspace, *cliPath) require.NoError(err) k, err := kubectl.New() @@ -79,10 +69,10 @@ func TestUpgrade(t *testing.T) { cli, err := getCLIPath(*cliPath) require.NoError(err) - targetVersions := writeUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices) + targetVersions := WriteUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices, constants.ConfigFilename) log.Println("Fetching measurements for new image.") - cmd := exec.CommandContext(context.Background(), cli, "config", "fetch-measurements", "--insecure", "--debug") + cmd := exec.CommandContext(t.Context(), cli, "config", "fetch-measurements", "--insecure", "--debug") stdout, stderr, err := runCommandWithSeparateOutputs(cmd) require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr)) log.Println(string(stdout)) @@ -92,82 +82,13 @@ func TestUpgrade(t *testing.T) { log.Println(string(data)) log.Println("Checking upgrade.") - runUpgradeCheck(require, cli, *targetKubernetes) + assert := assert.New(t) // use assert because this part is more brittle and should not fail the entire test + runUpgradeCheck(t.Context(), assert, cli, *targetKubernetes) log.Println("Triggering upgrade.") - runUpgradeApply(require, cli) + runUpgradeApply(t.Context(), require, cli) - wg := queryStatusAsync(t, cli) - - testMicroservicesEventuallyHaveVersion(t, targetVersions.microservices, *timeout) - testNodesEventuallyHaveVersion(t, k, targetVersions, *wantControl+*wantWorker, *timeout) - - wg.Wait() -} - -// setup checks that the prerequisites for the test are met: -// - a workspace is set -// - a CLI path is set -// - the constellation-upgrade folder does not exist. -func setup() error { - workingDir, err := workingDir(*workspace) - if err != nil { - return fmt.Errorf("getting working directory: %w", err) - } - - if err := os.Chdir(workingDir); err != nil { - return fmt.Errorf("changing working directory: %w", err) - } - - if _, err := getCLIPath(*cliPath); err != nil { - return fmt.Errorf("getting CLI path: %w", err) - } - return nil -} - -// workingDir returns the path to the workspace. -func workingDir(workspace string) (string, error) { - workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") - switch { - case workingDir != "": - return workingDir, nil - case workspace != "": - return workspace, nil - default: - return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") - } -} - -// getCLIPath returns the path to the CLI. -func getCLIPath(cliPathFlag string) (string, error) { - pathCLI := os.Getenv("PATH_CLI") - var relCLIPath string - switch { - case pathCLI != "": - relCLIPath = pathCLI - case cliPathFlag != "": - relCLIPath = cliPathFlag - default: - return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") - } - - // try to find the CLI in the working directory - // (e.g. when running via `go test` or when specifying a path manually) - workdir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("getting working directory: %w", err) - } - - absCLIPath := relCLIPath - if !filepath.IsAbs(relCLIPath) { - absCLIPath = filepath.Join(workdir, relCLIPath) - } - if _, err := os.Stat(absCLIPath); err == nil { - return absCLIPath, nil - } - - // fall back to runfiles (e.g. when running via bazel) - return runfiles.Rlocation(pathCLI) + AssertUpgradeSuccessful(t, cli, targetVersions, k, *wantControl, *wantWorker, *timeout) } // testPodsEventuallyReady checks that: @@ -175,7 +96,7 @@ func getCLIPath(cliPathFlag string) (string, error) { // 2) all pods have good status conditions. func testPodsEventuallyReady(t *testing.T, k *kubernetes.Clientset, namespace string) { require.Eventually(t, func() bool { - pods, err := k.CoreV1().Pods(namespace).List(context.Background(), metaV1.ListOptions{}) + pods, err := k.CoreV1().Pods(namespace).List(t.Context(), metaV1.ListOptions{}) if err != nil { log.Println(err) return false @@ -206,7 +127,7 @@ func testPodsEventuallyReady(t *testing.T, k *kubernetes.Clientset, namespace st // 2) the expected number of nodes have joined the cluster. func testNodesEventuallyAvailable(t *testing.T, k *kubernetes.Clientset, wantControlNodeCount, wantWorkerNodeCount int) { require.Eventually(t, func() bool { - nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{}) + nodes, err := k.CoreV1().Nodes().List(t.Context(), metaV1.ListOptions{}) if err != nil { log.Println(err) return false @@ -249,79 +170,27 @@ func testNodesEventuallyAvailable(t *testing.T, k *kubernetes.Clientset, wantCon }, time.Minute*30, time.Minute) } -func writeUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string) versionContainer { - fileHandler := file.NewHandler(afero.NewOsFs()) - attestationFetcher := attestationconfigapi.NewFetcher() - cfg, err := config.New(fileHandler, constants.ConfigFilename, attestationFetcher, true) - var cfgErr *config.ValidationError - var longMsg string - if errors.As(err, &cfgErr) { - longMsg = cfgErr.LongMessage() - } - require.NoError(err, longMsg) - - imageFetcher := imagefetcher.New() - imageRef, err := imageFetcher.FetchReference( - context.Background(), - cfg.GetProvider(), - cfg.GetAttestationConfig().GetVariant(), - image, - cfg.GetRegion(), cfg.UseMarketplaceImage(), - ) - require.NoError(err) - - log.Printf("Setting image version: %s\n", image) - cfg.Image = image - - defaultConfig := config.Default() - var kubernetesVersion versions.ValidK8sVersion - if kubernetes == "" { - kubernetesVersion = defaultConfig.KubernetesVersion - } else { - kubernetesVersion = versions.ValidK8sVersion(kubernetes) // ignore validation because the config is only written to file - } - - var microserviceVersion semver.Semver - if microservices == "" { - microserviceVersion = defaultConfig.MicroserviceVersion - } else { - version, err := semver.New(microservices) - require.NoError(err) - microserviceVersion = version - } - - log.Printf("Setting K8s version: %s\n", kubernetesVersion) - cfg.KubernetesVersion = kubernetesVersion - log.Printf("Setting microservice version: %s\n", microserviceVersion) - cfg.MicroserviceVersion = microserviceVersion - - err = fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptOverwrite) - require.NoError(err) - - return versionContainer{imageRef: imageRef, kubernetes: kubernetesVersion, microservices: microserviceVersion} -} - // runUpgradeCheck executes 'upgrade check' and does basic checks on the output. // We can not check images upgrades because we might use unpublished images. CLI uses public CDN to check for available images. -func runUpgradeCheck(require *require.Assertions, cli, targetKubernetes string) { - cmd := exec.CommandContext(context.Background(), cli, "upgrade", "check", "--debug") +func runUpgradeCheck(ctx context.Context, assert *assert.Assertions, cli, targetKubernetes string) { + cmd := exec.CommandContext(ctx, cli, "upgrade", "check", "--debug") stdout, stderr, err := runCommandWithSeparateOutputs(cmd) - require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr)) + assert.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr)) - require.Contains(string(stdout), "The following updates are available with this CLI:") - require.Contains(string(stdout), "Kubernetes:") + assert.Contains(string(stdout), "The following updates are available with this CLI:") + assert.Contains(string(stdout), "Kubernetes:") log.Printf("targetKubernetes: %s\n", targetKubernetes) if targetKubernetes == "" { log.Printf("true\n") - require.True(containsAny(string(stdout), versions.SupportedK8sVersions())) + assert.True(containsAny(string(stdout), versions.SupportedK8sVersions())) } else { log.Printf("false. targetKubernetes: %s\n", targetKubernetes) - require.Contains(string(stdout), targetKubernetes, fmt.Sprintf("Expected Kubernetes version %s in output.", targetKubernetes)) + assert.Contains(string(stdout), targetKubernetes, fmt.Sprintf("Expected Kubernetes version %s in output.", targetKubernetes)) } - require.Contains(string(stdout), "Services:") - require.Contains(string(stdout), fmt.Sprintf("--> %s", constants.BinaryVersion().String())) + assert.Contains(string(stdout), "Services:") + assert.Contains(string(stdout), fmt.Sprintf("--> %s", constants.BinaryVersion().String())) log.Println(string(stdout)) } @@ -335,16 +204,16 @@ func containsAny(text string, substrs []string) bool { return false } -func runUpgradeApply(require *require.Assertions, cli string) { +func runUpgradeApply(ctx context.Context, require *require.Assertions, cli string) { tfLogFlag := "" - cmd := exec.CommandContext(context.Background(), cli, "--help") + cmd := exec.CommandContext(ctx, cli, "--help") stdout, stderr, err := runCommandWithSeparateOutputs(cmd) require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr)) if strings.Contains(string(stdout), "--tf-log") { tfLogFlag = "--tf-log=DEBUG" } - cmd = exec.CommandContext(context.Background(), cli, "apply", "--debug", "--yes", tfLogFlag) + cmd = exec.CommandContext(ctx, cli, "apply", "--debug", "--yes", tfLogFlag) stdout, stderr, err = runCommandWithSeparateOutputs(cmd) require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr)) require.NoError(containsUnexepectedMsg(string(stdout))) @@ -361,140 +230,3 @@ func containsUnexepectedMsg(input string) error { } return nil } - -func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - // The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy. - // Nodes can upgrade in <5mins. - testStatusEventuallyWorks(t, cli, 20*time.Minute) - }() - - return &wg -} - -func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) { - require.Eventually(t, func() bool { - // Show versions set in cluster. - // The string after "Cluster status:" in the output might not be updated yet. - // This is only updated after the operator finishes one reconcile loop. - cmd := exec.CommandContext(context.Background(), cli, "status") - stdout, stderr, err := runCommandWithSeparateOutputs(cmd) - if err != nil { - log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr)) - return false - } - - log.Println(string(stdout)) - return true - }, timeout, time.Minute) -} - -func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) { - require.Eventually(t, func() bool { - version, err := servicesVersion(t) - if err != nil { - log.Printf("Unable to fetch microservice version: %v\n", err) - return false - } - - if version != wantMicroserviceVersion { - log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion) - return false - } - - return true - }, timeout, time.Minute) -} - -func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions versionContainer, totalNodeCount int, timeout time.Duration) { - require.Eventually(t, func() bool { - nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{}) - if err != nil { - log.Println(err) - return false - } - require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items)) - - allUpdated := true - log.Printf("Node status (%v):", time.Now()) - for _, node := range nodes.Items { - for key, value := range node.Annotations { - if key == "constellation.edgeless.systems/node-image" { - if !strings.EqualFold(value, targetVersions.imageRef) { - log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.imageRef) - allUpdated = false - } - } - } - - kubeletVersion := node.Status.NodeInfo.KubeletVersion - if kubeletVersion != string(targetVersions.kubernetes) { - log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.kubernetes) - allUpdated = false - } - kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion - if kubeProxyVersion != string(targetVersions.kubernetes) { - log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.kubernetes) - allUpdated = false - } - } - - return allUpdated - }, timeout, time.Minute) -} - -type versionContainer struct { - imageRef string - kubernetes versions.ValidK8sVersion - microservices semver.Semver -} - -// runCommandWithSeparateOutputs runs the given command while separating buffers for -// stdout and stderr. -func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) { - stdout = []byte{} - stderr = []byte{} - - stdoutIn, err := cmd.StdoutPipe() - if err != nil { - err = fmt.Errorf("create stdout pipe: %w", err) - return - } - stderrIn, err := cmd.StderrPipe() - if err != nil { - err = fmt.Errorf("create stderr pipe: %w", err) - return - } - - err = cmd.Start() - if err != nil { - err = fmt.Errorf("start command: %w", err) - return - } - - continuouslyPrintOutput := func(r io.Reader, prefix string) { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - output := scanner.Text() - fmt.Printf("%s: %s\n", prefix, output) - switch prefix { - case "stdout": - stdout = append(stdout, output...) - case "stderr": - stderr = append(stderr, output...) - } - } - } - - go continuouslyPrintOutput(stdoutIn, "stdout") - go continuouslyPrintOutput(stderrIn, "stderr") - - if err = cmd.Wait(); err != nil { - err = fmt.Errorf("wait for command to finish: %w", err) - } - - return stdout, stderr, err -} diff --git a/e2e/malicious-join/BUILD.bazel b/e2e/malicious-join/BUILD.bazel index 0075cdf45..6004b41bb 100644 --- a/e2e/malicious-join/BUILD.bazel +++ b/e2e/malicious-join/BUILD.bazel @@ -16,8 +16,6 @@ go_library( "//internal/grpc/dialer", "//internal/logger", "//joinservice/joinproto", - "@org_uber_go_zap//:zap", - "@org_uber_go_zap//zapcore", ], ) @@ -67,7 +65,8 @@ genrule( oci_push( name = "malicious-join_push", image = ":malicious-join_image", - repotags = ":repotag.txt", + remote_tags = "//bazel/settings:tag", + repository_file = ":container_name", ) sh_template( diff --git a/e2e/malicious-join/malicious-join.go b/e2e/malicious-join/malicious-join.go index a8894d023..c32885af3 100644 --- a/e2e/malicious-join/malicious-join.go +++ b/e2e/malicious-join/malicious-join.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // End-to-end test that issues various types of malicious join requests to a cluster. @@ -12,7 +12,9 @@ import ( "encoding/json" "flag" "fmt" + "log/slog" "net" + "os" "strings" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" @@ -20,13 +22,10 @@ import ( "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/joinservice/joinproto" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" ) func main() { - log := logger.New(logger.JSONLog, zapcore.DebugLevel) - defer log.Sync() + log := logger.NewJSONLogger(slog.LevelDebug) jsEndpoint := flag.String("js-endpoint", "", "Join service endpoint to use.") csp := flag.String("csp", "", "Cloud service provider to use.") @@ -38,13 +37,13 @@ func main() { ) flag.Parse() log.With( - zap.String("js-endpoint", *jsEndpoint), - zap.String("csp", *csp), - zap.String("variant", *attVariant), - ).Infof("Running tests with flags") + slog.String("js-endpoint", *jsEndpoint), + slog.String("csp", *csp), + slog.String("variant", *attVariant), + ).Info("Running tests with flags") testCases := map[string]struct { - fn func(attVariant, csp, jsEndpoint string, log *logger.Logger) error + fn func(attVariant, csp, jsEndpoint string, log *slog.Logger) error wantErr bool }{ "JoinFromUnattestedNode": { @@ -58,44 +57,45 @@ func main() { TestCases: make(map[string]testCaseOutput), } for name, tc := range testCases { - log.With(zap.String("testcase", name)).Infof("Running testcase") + log.With(slog.String("testcase", name)).Info("Running testcase") err := tc.fn(*attVariant, *csp, *jsEndpoint, log) switch { case err == nil && tc.wantErr: - log.With(zap.Error(err), zap.String("testcase", name)).Errorf("Test case failed: Expected error but got none") + log.With(slog.Any("error", err), slog.String("testcase", name)).Error("Test case failed: Expected error but got none") testOutput.TestCases[name] = testCaseOutput{ Passed: false, Message: "Expected error but got none", } allPassed = false case !tc.wantErr && err != nil: - log.With(zap.Error(err), zap.String("testcase", name)).Errorf("Test case failed: Got unexpected error") + log.With(slog.Any("error", err), slog.String("testcase", name)).Error("Test case failed: Got unexpected error") testOutput.TestCases[name] = testCaseOutput{ Passed: false, Message: fmt.Sprintf("Got unexpected error: %s", err), } allPassed = false case tc.wantErr && err != nil: - log.With(zap.String("testcase", name)).Infof("Test case succeeded") + log.With(slog.String("testcase", name)).Info("Test case succeeded") testOutput.TestCases[name] = testCaseOutput{ Passed: true, Message: fmt.Sprintf("Got expected error: %s", err), } case !tc.wantErr && err == nil: - log.With(zap.String("testcase", name)).Infof("Test case succeeded") + log.With(slog.String("testcase", name)).Info("Test case succeeded") testOutput.TestCases[name] = testCaseOutput{ Passed: true, Message: "No error, as expected", } default: - log.With(zap.String("testcase", name)).Fatalf("invalid result") + log.With(slog.String("testcase", name)).Error("invalid result") + os.Exit(1) } } testOutput.AllPassed = allPassed - log.With(zap.Any("result", testOutput)).Infof("Test completed") + log.With(slog.Any("result", testOutput)).Info("Test completed") } type testOutput struct { @@ -110,7 +110,7 @@ type testCaseOutput struct { // JoinFromUnattestedNode simulates a join request from a Node that uses a stub issuer // and thus cannot be attested correctly. -func JoinFromUnattestedNode(attVariant, csp, jsEndpoint string, log *logger.Logger) error { +func JoinFromUnattestedNode(attVariant, csp, jsEndpoint string, log *slog.Logger) error { joiner, err := newMaliciousJoiner(attVariant, csp, jsEndpoint, log) if err != nil { return fmt.Errorf("creating malicious joiner: %w", err) @@ -125,7 +125,7 @@ func JoinFromUnattestedNode(attVariant, csp, jsEndpoint string, log *logger.Logg // newMaliciousJoiner creates a new malicious joiner, i.e. a simulated node that issues // an invalid join request. -func newMaliciousJoiner(attVariant, csp, endpoint string, log *logger.Logger) (*maliciousJoiner, error) { +func newMaliciousJoiner(attVariant, csp, endpoint string, log *slog.Logger) (*maliciousJoiner, error) { var attVariantOid variant.Variant var err error if strings.EqualFold(attVariant, "default") { @@ -149,33 +149,33 @@ func newMaliciousJoiner(attVariant, csp, endpoint string, log *logger.Logger) (* // maliciousJoiner simulates a malicious node joining a cluster. type maliciousJoiner struct { endpoint string - logger *logger.Logger + logger *slog.Logger dialer *dialer.Dialer } // join issues a join request to the join service endpoint. func (j *maliciousJoiner) join(ctx context.Context) (*joinproto.IssueJoinTicketResponse, error) { - j.logger.Debugf("Dialing join service endpoint %s", j.endpoint) - conn, err := j.dialer.Dial(ctx, j.endpoint) + j.logger.Debug(fmt.Sprintf("Dialing join service endpoint %q", j.endpoint)) + conn, err := j.dialer.Dial(j.endpoint) if err != nil { return nil, fmt.Errorf("dialing join service endpoint: %w", err) } defer conn.Close() - j.logger.Debugf("Successfully dialed join service endpoint %s", j.endpoint) + j.logger.Debug(fmt.Sprintf("Successfully dialed join service endpoint %q", j.endpoint)) protoClient := joinproto.NewAPIClient(conn) - j.logger.Debugf("Issuing join ticket") + j.logger.Debug("Issuing join ticket") req := &joinproto.IssueJoinTicketRequest{ DiskUuid: "", CertificateRequest: []byte{}, IsControlPlane: false, } res, err := protoClient.IssueJoinTicket(ctx, req) - j.logger.Debugf("Got join ticket response: %+v", res) if err != nil { return nil, fmt.Errorf("issuing join ticket: %w", err) } + j.logger.Debug("Got join ticket response", "apiServerEndpoint", res.ApiServerEndpoint, "kubernetesVersion", res.KubernetesVersion) return res, nil } diff --git a/e2e/miniconstellation/.terraform.lock.hcl b/e2e/miniconstellation/.terraform.lock.hcl index c110babaf..87cddd3bb 100644 --- a/e2e/miniconstellation/.terraform.lock.hcl +++ b/e2e/miniconstellation/.terraform.lock.hcl @@ -2,103 +2,91 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.74.0" - constraints = "3.74.0" + version = "4.29.0" + constraints = "4.29.0" hashes = [ - "h1:1kSiowd/tBNswp3iv7ePlzkP5llWihjHcY3pdXdJqVU=", - "h1:4b15khHtc5OkIVEFg0W5QRwf/ov1WVQkXVdSiAcTCS8=", - "h1:ETVZfmulZQ435+lgFCkZRpfVOLyAxfDOwbPXFg3aLLQ=", - "h1:H3diAufZ5VDQKsQNYykVRaFTOUJ4gjFiT2VLYi574+w=", - "h1:LEdK8BxNSNiBQbtcJhQZKMMHDjmPpUsvDpr3Mzs93Tg=", - "h1:OtJKZcMwrRNR84ylT1GgMwGR8KTxVOCkNifbjABlGj0=", - "h1:Rq+CNb+4u47dw20tlAeI2yxSOuDtLm+S/GZO2pneLyA=", - "h1:VfBB00BE0wvFiod7BlL+Cn6r2599MEi94hnAQ277ux8=", - "h1:YJ15rwD0G7lYc9OVh5GO4VTqcd2jhqegfgyqTJH1M/I=", - "h1:YvxxiqiwXjZdU53u3b9q49ezsIAb59KmdLLFkwkwFAs=", - "h1:xDRmcV40KrWttPYg/w0/IN/frS9K1twuyvqRNVZko44=", - "zh:0424c70152f949da1ec52ba96d20e5fd32fd22d9bd9203ce045d5f6aab3d20fc", - "zh:16dbf581d10f8e7937185bcdcceb4f91d08c919e452fb8da7580071288c8c397", - "zh:3019103bc2c3b4e185f5c65696c349697644c968f5c085af5505fed6d01c4241", - "zh:49bb56ebaed6653fdb913c2b2bb74fc8b5399e7258d1e89084f72c44ea1130dd", - "zh:85547666517f899d88620bd23a000a8f43c7dc93587c350eb1ea17bcb3e645c7", - "zh:8bed8b646ff1822d8764de68b56b71e5dd971a4b77eba80d47f400a530800bea", - "zh:8bfa6c70c004ba05ebce47f74f49ce872c28a68a18bb71b281a9681bcbbdbfa1", - "zh:a2ae9e38fda0695fb8aa810e4f1ce4b104bfda651a87923b307bb1728680d8b6", - "zh:beac1efe32f99072c892095f5ff46e40d6852b66679a03bc3acbe1b90fb1f653", - "zh:d8a6ca20e49ebe7ea5688d91233d571e2c2ccc3e41000c39a7d7031df209ea8e", + "h1:Bde/KCh2xGVCBx/JnixC3I2fmoRTwHXgsapfQ5QG8eg=", + "h1:IyINmgNiLfWx3Istkt5Mz+IJrDhSMhj3/qQeJlC4qS4=", + "h1:KEJAt0mJAACyIKUB5mCk/wqtxKMhivdeW8w6byz5Ll4=", + "h1:Y4gTSs+ZE5YSJVXG2qmsbXmv9Daq5aGM8Ip/GE6nev8=", + "h1:YtcHvTVfVBKbMCp9esoj527R1UK/hU0Zmo3pyQb8YhQ=", + "h1:atJdgnuqk+w3v4Zzhw2B1FZeYYA4su9JfanwNsx+c8o=", + "h1:c9tmtEdVTb9siGa3hVxPrMVl9ij5zijnD02JMHcHjrE=", + "h1:eN0KhMGVepEPpSA+YN5Kaz/v9PFKCafbkqqBzpLJf+g=", + "h1:hNVKlXTx2duXnR6SNKtyQMx7zSIlrxBu66Z0gbyfv3c=", + "h1:jC2GJo4VzTKnKociUDLVv8/+u9Mz+4scZrqbEasV+Y0=", + "h1:m3xYvc9X0pec0Zd1dpn82ALQ+6vwz56RnF/3CbkI2Eg=", + "zh:16590eea68c7c8aedb7af19f690eb34ab6636ef417b3fa9e06ca038fdb4c44b8", + "zh:1c907dfe44d00a54aa63d443004add90188f9a53ef3e919aff8aba92f715f42b", + "zh:258a0ff4198d80cae33c89091cd556d84c1b522c4416458484f23719a0cdf4cc", + "zh:587f5e9b2b33e51b18fb0f372025c961c3f57f2958b388459dd8432412650bda", + "zh:6318ca03bd9dbac272a75bb193688c7d4c4b45c7460289820528f31bcd6c3fe9", + "zh:63e4e8128e26e4c3e0c3b6582ef527245eb35eb5c80ad278dc675ebdf71edeaf", + "zh:845c898a27a84a15ba26e95ee66ac9563f81bc672b5ca216af82d87fe09bd5f8", + "zh:8fa6434fa212d5501185f0adc985d3a3c1e0f449c78f040a4ca640cb1e809cac", + "zh:9b49c0d72ab19aab43b2b48d23c5dddbbe29afae1569a987e6f20ed4c80ddf4c", + "zh:b14cc1ee5e3acf52490de7dd9791cce7953c0ee4bcccf0306aafd256568bd69f", + "zh:cd444836b2579fa42bfca2ae6145d394c41b6438b1ae01078c060bfaf803bb4d", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f937b5fdf49b072c0347408d0a1c5a5d822dae1a23252915930e5a82d1d8ce8b", ] } provider "registry.terraform.io/hashicorp/cloudinit" { - version = "2.3.2" - constraints = "2.3.2" + version = "2.3.7" + constraints = "2.3.7" hashes = [ - "h1:2jb+BfT5T96dXxUD2LQ6MtVHpXErd7ZybmMvdWE2jd4=", - "h1:Ar/DAbZQ9Nsj0BrqX6camrEE6U+Yq4E87DCNVqxqx8k=", - "h1:Vl0aixAYTV/bjathX7VArC5TVNkxBCsi3Vq7R4z1uvc=", - "h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=", - "h1:y+6FsU2STOpx6L6JOon4DVZoZPQgNoR2xR2WQ/EVxcQ=", - "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", - "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", - "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", - "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f", - "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422", - "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372", + "h1:/hny5kXmhcnuJDD1V+5XCrZOYDIqja2U47VM4DPEnBA=", + "h1:A9COAUjeBJ+fgYAI/PKtDs4Wzs50srFSY+KkfpSVGLw=", + "h1:Lt8lqrdNgZRlkOTwSXZTyuJkiVXnpwTsWAqHQPL6sIY=", + "h1:M9TpQxKAE/hyOwytdX9MUNZw30HoD/OXqYIug5fkqH8=", + "h1:coZHiZww6hWZoOoWw0p+6oeYb/tMh1uTvX1Y2ZzzXqE=", + "h1:dgBaiMxxU61piW30emM6251LMFW66TbKR+p5ylPZvqc=", + "h1:h1Pr6uNwq+iDEGrnQJEHzOTz+yVTW0AJgZrGXuoO4Qs=", + "h1:ht83gEvyri0BD3sata7BDhx31N/KbCECIozG7UM/kC8=", + "h1:iZ27qylcH/2bs685LJTKOKcQ+g7cF3VwN3kHMrzm4Ow=", + "h1:ll35IR++uaXwfwqZFFRWrvS0idO1mX43Y/embsaOe4k=", + "h1:rafNPmTutVTO2Horq45DG9Pjqrs+vx42oc7b/3aVGEc=", + "zh:06f1c54e919425c3139f8aeb8fcf9bceca7e560d48c9f0c1e3bb0a8ad9d9da1e", + "zh:0e1e4cf6fd98b019e764c28586a386dc136129fef50af8c7165a067e7e4a31d5", + "zh:1871f4337c7c57287d4d67396f633d224b8938708b772abfc664d1f80bd67edd", + "zh:2b9269d91b742a71b2248439d5e9824f0447e6d261bfb86a8a88528609b136d1", + "zh:3d8ae039af21426072c66d6a59a467d51f2d9189b8198616888c1b7fc42addc7", + "zh:3ef4e2db5bcf3e2d915921adced43929214e0946a6fb11793085d9a48995ae01", + "zh:42ae54381147437c83cbb8790cc68935d71b6357728a154109d3220b1beb4dc9", + "zh:4496b362605ae4cbc9ef7995d102351e2fe311897586ffc7a4a262ccca0c782a", + "zh:652a2401257a12706d32842f66dac05a735693abcb3e6517d6b5e2573729ba13", + "zh:7406c30806f5979eaed5f50c548eced2ea18ea121e01801d2f0d4d87a04f6a14", + "zh:7848429fd5a5bcf35f6fee8487df0fb64b09ec071330f3ff240c0343fe2a5224", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136", - "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7", - "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc", - "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040", - "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.6.0" - constraints = "3.6.0" - hashes = [ - "h1:5KeoKSVKVHJW308uiTgslxFbjQAdWzBGUFK68vgMRWY=", - "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", - "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=", - "h1:p6WG1IPHnqx1fnJVKNjv733FBaArIugqy58HRZnpPCk=", - "h1:t0mRdJzegohRKhfdoQEJnv3JRISSezJRblN0HIe67vo=", - "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", - "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", - "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", - "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", - "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", - "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", - "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", - "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", - "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", - "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", ] } provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.4" - constraints = "4.0.4" + version = "4.1.0" + constraints = "4.1.0" hashes = [ - "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=", - "h1:Wd3RqmQW60k2QWPN4sK5CtjGuO1d+CRNXgC+D4rKtXc=", - "h1:bNsvpX5EGuVxgGRXBQVLXlmq40PdoLp8Rfuh1ZmV7yY=", - "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", - "h1:rKKMyIEBZwR+8j6Tx3PwqBrStuH+J+pxcbCR5XN8WAw=", - "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", - "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", - "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", - "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", - "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", - "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", - "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", - "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", - "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", - "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", - "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "h1:4gd/jiOS0zJxjTd5Q4o/gOp24RxcuwQ/TxwjTYQNPz4=", + "h1:C0J7AsrVHVqnDT9tICDNaKvA9iH6WTLS2EYzCEegpx0=", + "h1:Ka8mEwRFXBabR33iN/WTIEW6RP0z13vFsDlwn11Pf2I=", + "h1:ReNkTkCM64bktu54eGwQc29rhIejMLQsYA6kYNyBWno=", + "h1:UklaKJOCynnEJbpCVN0zJKIJ3SvO7RQJ00/6grBatnw=", + "h1:ZHcr1WIomuU6ZV+dzEwAG1+52JP0e0d/+l7bo3N5p88=", + "h1:eZa3vbx1pbiwnajuKvGWE7jWK+nHQ8lcLc/mO6Rhf4o=", + "h1:iSgnCUoLGMkt31RlflnL09NyjpAH0DX6bb9QBw5IE9Y=", + "h1:uDtqTpFJOseNUlPDx4TT/lXf6ie3CarsimL7sYCiVH4=", + "h1:y9cHrgcuaZt592In6xQzz1lx7k/B9EeWrAb8K7QqOgU=", + "h1:zEv9tY1KR5vaLSyp2lkrucNJ+Vq3c+sTFK9GyQGLtFs=", + "zh:14c35d89307988c835a7f8e26f1b83ce771e5f9b41e407f86a644c0152089ac2", + "zh:2fb9fe7a8b5afdbd3e903acb6776ef1be3f2e587fb236a8c60f11a9fa165faa8", + "zh:35808142ef850c0c60dd93dc06b95c747720ed2c40c89031781165f0c2baa2fc", + "zh:35b5dc95bc75f0b3b9c5ce54d4d7600c1ebc96fbb8dfca174536e8bf103c8cdc", + "zh:38aa27c6a6c98f1712aa5cc30011884dc4b128b4073a4a27883374bfa3ec9fac", + "zh:51fb247e3a2e88f0047cb97bb9df7c228254a3b3021c5534e4563b4007e6f882", + "zh:62b981ce491e38d892ba6364d1d0cdaadcee37cc218590e07b310b1dfa34be2d", + "zh:bc8e47efc611924a79f947ce072a9ad698f311d4a60d0b4dfff6758c912b7298", + "zh:c149508bd131765d1bc085c75a870abb314ff5a6d7f5ac1035a8892d686b6297", + "zh:d38d40783503d278b63858978d40e07ac48123a2925e1a6b47e62179c046f87a", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb07f708e3316615f6d218cec198504984c0ce7000b9f1eebff7516e384f4b54", ] } diff --git a/e2e/miniconstellation/BUILD.bazel b/e2e/miniconstellation/BUILD.bazel index 836b2d075..47dd3b278 100644 --- a/e2e/miniconstellation/BUILD.bazel +++ b/e2e/miniconstellation/BUILD.bazel @@ -1,4 +1,5 @@ load("@com_github_ash2k_bazel_tools//multirun:def.bzl", "multirun") +load("@rules_shell//shell:sh_library.bzl", "sh_library") load("//bazel/sh:def.bzl", "sh_template") filegroup( @@ -9,6 +10,7 @@ filegroup( "main.tf", "output.tf", "test-remote.sh", + "variables.tf", ], ) diff --git a/e2e/miniconstellation/main.sh.in b/e2e/miniconstellation/main.sh.in index 694f17bb3..f25975554 100755 --- a/e2e/miniconstellation/main.sh.in +++ b/e2e/miniconstellation/main.sh.in @@ -23,6 +23,11 @@ cd e2e/miniconstellation echo "::group::Terraform" +random_string=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 6) +rg_name="e2e-mini-${random_string}" +echo "rgname=${rg_name}" >> "${GITHUB_OUTPUT:-/dev/null}" +echo "resource_name = \"${rg_name}\"" > terraform.tfvars + terraform init terraform apply -auto-approve terraform output -raw ssh_private_key > id_rsa diff --git a/e2e/miniconstellation/main.tf b/e2e/miniconstellation/main.tf index e5181c7c1..2054a63a9 100644 --- a/e2e/miniconstellation/main.tf +++ b/e2e/miniconstellation/main.tf @@ -2,19 +2,15 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.74.0" - } - random = { - source = "hashicorp/random" - version = "3.6.0" + version = "4.29.0" } tls = { source = "hashicorp/tls" - version = "4.0.4" + version = "4.1.0" } cloudinit = { source = "hashicorp/cloudinit" - version = "2.3.2" + version = "2.3.7" } } } @@ -22,15 +18,13 @@ terraform { provider "azurerm" { use_oidc = true features {} + # This enables all resource providers. + # In the future, we might want to use `resource_providers_to_register` to registers just the ones we need. + resource_provider_registrations = "all" } provider "tls" {} -resource "random_string" "suffix" { - length = 6 - special = false -} - resource "tls_private_key" "ssh_key" { algorithm = "RSA" rsa_bits = 2048 @@ -46,36 +40,37 @@ data "cloudinit_config" "cloud_init" { } } -data "azurerm_resource_group" "main" { - name = "e2e-miniconstellation" +resource "azurerm_resource_group" "main" { + name = var.resource_name + location = "West Europe" } resource "azurerm_virtual_network" "main" { - name = "e2e-mini-${random_string.suffix.result}" + name = var.resource_name address_space = ["10.0.0.0/16"] - location = data.azurerm_resource_group.main.location - resource_group_name = data.azurerm_resource_group.main.name + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name } resource "azurerm_subnet" "main" { - name = "e2e-mini-${random_string.suffix.result}" - resource_group_name = data.azurerm_resource_group.main.name + name = var.resource_name + resource_group_name = azurerm_resource_group.main.name virtual_network_name = azurerm_virtual_network.main.name address_prefixes = ["10.0.2.0/24"] } resource "azurerm_public_ip" "main" { - name = "e2e-mini-${random_string.suffix.result}" - location = data.azurerm_resource_group.main.location - resource_group_name = data.azurerm_resource_group.main.name + name = var.resource_name + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name allocation_method = "Static" sku = "Standard" } resource "azurerm_network_interface" "main" { - name = "e2e-mini-${random_string.suffix.result}" - resource_group_name = data.azurerm_resource_group.main.name - location = data.azurerm_resource_group.main.location + name = var.resource_name + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location ip_configuration { name = "main" @@ -86,9 +81,9 @@ resource "azurerm_network_interface" "main" { } resource "azurerm_network_security_group" "ssh" { - name = "e2e-mini-${random_string.suffix.result}" - resource_group_name = data.azurerm_resource_group.main.name - location = data.azurerm_resource_group.main.location + name = var.resource_name + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location security_rule { name = "ssh" @@ -109,9 +104,9 @@ resource "azurerm_subnet_network_security_group_association" "ssh" { } resource "azurerm_linux_virtual_machine" "main" { - name = "e2e-mini-${random_string.suffix.result}" - resource_group_name = data.azurerm_resource_group.main.name - location = data.azurerm_resource_group.main.location + name = var.resource_name + resource_group_name = azurerm_resource_group.main.name + location = azurerm_resource_group.main.location # Standard_D8s_v5 provides nested virtualization support size = "Standard_D8s_v5" diff --git a/e2e/miniconstellation/test-remote.sh b/e2e/miniconstellation/test-remote.sh index 7166dec1c..c5fbf4ac5 100755 --- a/e2e/miniconstellation/test-remote.sh +++ b/e2e/miniconstellation/test-remote.sh @@ -45,24 +45,35 @@ fi echo "Done waiting." +echo '127.0.0.1 license.confidential.cloud' | sudo tee /etc/hosts > /dev/null + ./constellation mini up --debug export KUBECONFIG="${PWD}/constellation-admin.conf" -# Wait for nodes to actually show up in K8s -count=0 -until kubectl wait --for=condition=Ready --timeout=2s nodes control-plane-0 2> /dev/null || [[ ${count} -eq 30 ]]; do - echo "Control-planes are not registered in Kubernetes yet. Waiting..." - sleep 10 - count=$((count + 1)) -done - -count=0 -until kubectl wait --for=condition=Ready --timeout=2s nodes worker-0 2> /dev/null || [[ ${count} -eq 30 ]]; do - echo "Worker nodes are not registered in Kubernetes yet. Waiting..." - sleep 10 - count=$((count + 1)) +# Wait for nodes to actually show up in K8s (taken from .github/actions/constellation_create/action.yml) +echo "::group::Wait for nodes" +NODES_COUNT=2 +JOINWAIT=0 +JOINTIMEOUT="600" # 10 minutes timeout for all nodes to join +until [[ "$(kubectl get nodes -o json | jq '.items | length')" == "${NODES_COUNT}" ]] || [[ $JOINWAIT -gt $JOINTIMEOUT ]]; do + echo "$(kubectl get nodes -o json | jq '.items | length')/${NODES_COUNT} nodes have joined.. waiting.." + JOINWAIT=$((JOINWAIT + 30)) + sleep 30 done +if [[ $JOINWAIT -gt $JOINTIMEOUT ]]; then + echo "Timed out waiting for nodes to join" + exit 1 +fi +echo "$(kubectl get nodes -o json | jq '.items | length')/${NODES_COUNT} nodes have joined" +if ! kubectl wait --for=condition=ready --all nodes --timeout=20m; then + kubectl get pods -n kube-system + kubectl get events -n kube-system + echo "::error::kubectl wait timed out before all nodes became ready" + echo "::endgroup::" + exit 1 +fi +echo "::endgroup::" # Wait for deployments kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deployment coredns @@ -71,7 +82,6 @@ kubectl -n kube-system wait --for=condition=Available=True --timeout=180s deploy kubectl -n kube-system rollout status --timeout 180s daemonset cilium kubectl -n kube-system rollout status --timeout 180s daemonset join-service kubectl -n kube-system rollout status --timeout 180s daemonset key-service -kubectl -n kube-system rollout status --timeout 180s daemonset konnectivity-agent kubectl -n kube-system rollout status --timeout 180s daemonset verification-service echo "Miniconstellation started successfully. Shutting down..." diff --git a/e2e/miniconstellation/variables.tf b/e2e/miniconstellation/variables.tf new file mode 100644 index 000000000..eb2034c7f --- /dev/null +++ b/e2e/miniconstellation/variables.tf @@ -0,0 +1,4 @@ +variable "resource_name" { + type = string + description = "name for resources to create" +} diff --git a/e2e/provider-upgrade/BUILD.bazel b/e2e/provider-upgrade/BUILD.bazel new file mode 100644 index 000000000..6feef04fb --- /dev/null +++ b/e2e/provider-upgrade/BUILD.bazel @@ -0,0 +1,16 @@ +load("//bazel/go:go_test.bzl", "go_test") + +go_test( + name = "provider-upgrade_test", + srcs = ["upgrade_test.go"], + # keep + count = 1, + gotags = ["e2e"], + tags = ["manual"], + deps = [ + "//e2e/internal/kubectl", + "//e2e/internal/upgrade", + "//internal/constants", + "@com_github_stretchr_testify//require", + ], +) diff --git a/e2e/provider-upgrade/upgrade_test.go b/e2e/provider-upgrade/upgrade_test.go new file mode 100644 index 000000000..821158ac6 --- /dev/null +++ b/e2e/provider-upgrade/upgrade_test.go @@ -0,0 +1,60 @@ +//go:build e2e + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +// End-to-end test that is used by the e2e Terraform provider test. +package main + +import ( + "flag" + "os" + "path/filepath" + "testing" + "time" + + "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" + "github.com/edgelesssys/constellation/v2/e2e/internal/upgrade" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/stretchr/testify/require" +) + +var ( + targetImage = flag.String("target-image", "", "Image (shortversion) to upgrade to.") + targetKubernetes = flag.String("target-kubernetes", "", "Kubernetes version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") + targetMicroservices = flag.String("target-microservices", "", "Microservice version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") + // When executing the test as a bazel target the CLI path is supplied through an env variable that bazel sets. + // When executing via `go test` extra care should be taken that the supplied CLI is built on the same commit as this test. + // When executing the test as a bazel target the workspace path is supplied through an env variable that bazel sets. + workspace = flag.String("workspace", "", "Constellation workspace in which to run the tests.") + cliPath = flag.String("cli", "", "Constellation CLI to run the tests.") + wantWorker = flag.Int("want-worker", 0, "Number of wanted worker nodes.") + wantControl = flag.Int("want-control", 0, "Number of wanted control nodes.") + timeout = flag.Duration("timeout", 90*time.Minute, "Timeout after which the cluster should have converged to the target version.") +) + +// TestUpgradeSuccessful tests that the upgrade to the target version is successful. +func TestUpgradeSuccessful(t *testing.T) { + require := require.New(t) + kubeconfigPath := os.Getenv("KUBECONFIG") + require.NotEmpty(kubeconfigPath, "KUBECONFIG environment variable must be set") + dir := filepath.Dir(kubeconfigPath) + configPath := filepath.Join(dir, constants.ConfigFilename) + + // only done here to construct the version struct + require.NotEqual(*targetImage, "", "--target-image needs to be specified") + v := upgrade.WriteUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices, configPath) + // ignore Kubernetes check if targetKubernetes is not set; Kubernetes is only explicitly upgraded + if *targetKubernetes == "" { + v.Kubernetes = "" + } + k, err := kubectl.New() + require.NoError(err) + + err = upgrade.Setup(*workspace, *cliPath) + require.NoError(err) + upgrade.AssertUpgradeSuccessful(t, *cliPath, v, k, *wantControl, *wantWorker, *timeout) +} diff --git a/flake.lock b/flake.lock index 297ea4ad2..b424b9c4a 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1701680307, - "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -18,13 +18,13 @@ "type": "github" } }, - "nixpkgsUnstable": { + "nixpkgs": { "locked": { - "lastModified": 1702206697, - "narHash": "sha256-vE9oEx3Y8TO5MnWwFlmopjHd1JoEBno+EhsfUCq5iR8=", + "lastModified": 1743938762, + "narHash": "sha256-UgFYn8sGv9B8PoFpUfCa43CjMZBl1x/ShQhRDHBFQdI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "29d6c96900b9b576c2fb89491452f283aa979819", + "rev": "74a40410369a1c35ee09b8a1abee6f4acbedc059", "type": "github" }, "original": { @@ -37,7 +37,7 @@ "root": { "inputs": { "flake-utils": "flake-utils", - "nixpkgsUnstable": "nixpkgsUnstable" + "nixpkgs": "nixpkgs" } }, "systems": { diff --git a/flake.nix b/flake.nix index f593777f3..1e1304460 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "Constellation"; inputs = { - nixpkgsUnstable = { + nixpkgs = { url = "github:NixOS/nixpkgs/nixpkgs-unstable"; }; flake-utils = { @@ -11,66 +11,126 @@ }; outputs = - { self - , nixpkgsUnstable - , flake-utils - }: - flake-utils.lib.eachDefaultSystem (system: - let - pkgsUnstable = import nixpkgsUnstable { inherit system; }; - - callPackage = pkgsUnstable.callPackage; - - mkosiDev = (pkgsUnstable.mkosi.overrideAttrs (oldAttrs: rec { - # TODO(malt3): remove patch once merged and released upstream (systemd/mkosi#2163) - src = pkgsUnstable.fetchFromGitHub { - owner = "systemd"; - repo = "mkosi"; - rev = "abf22cdc6ccb13f2cd84679ede77231455ec6813"; - hash = "sha256-njtYWSXSLMcn6AtGfAeL/ncZQ6g+Vgpe7EaKLkzAOl4="; - }; - propagatedBuildInputs = oldAttrs.propagatedBuildInputs ++ (with pkgsUnstable; [ - # package management - dnf5 - rpm - - # filesystem tools - squashfsTools # mksquashfs - dosfstools # mkfs.vfat - mtools # mcopy - cryptsetup # dm-verity - util-linux # flock - kmod # depmod - cpio # cpio - zstd # zstd - xz # xz - ]); - })); - - openssl-static = pkgsUnstable.openssl.override { static = true; }; - - in { - packages.mkosi = mkosiDev; + self, + nixpkgs, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + overlay = final: prev: { + rpm = prev.rpm.overrideAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ prev.makeWrapper ]; + postFixup = '' + wrapProgram $out/lib/rpm/sysusers.sh \ + --set PATH ${ + prev.lib.makeBinPath ( + with prev; + [ + coreutils + findutils + su.out + gnugrep + ] + ) + } + ''; + }); - packages.openssl = callPackage ./nix/cc/openssl.nix { pkgs = pkgsUnstable; }; + # dnf5 assumes a TTY with a very small width by default, truncating its output instead of line-wrapping + # it. Force it to use more VT columns to avoid this, and make debugging errors easier. + dnf5-stub = prev.writeScriptBin "dnf5" '' + #!/usr/bin/env bash + FORCE_COLUMNS=200 ${final.dnf5}/bin/dnf5 $@ + ''; + }; - packages.cryptsetup = callPackage ./nix/cc/cryptsetup.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; }; + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; - packages.libvirt = callPackage ./nix/cc/libvirt.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; }; + overlays = [ + (_final: prev: (import ./nix/packages { inherit (prev) lib callPackage; })) + (_final: prev: { lib = prev.lib // (import ./nix/lib { inherit (prev) lib callPackage; }); }) + overlay + ]; + }; - packages.libvirtd_base = callPackage ./nix/container/libvirtd_base.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; }; + callPackage = pkgs.callPackage; - packages.awscli2 = pkgsUnstable.awscli2; + mkosiDev = ( + pkgs.mkosi.override { + extraDeps = ( + with pkgs; + [ + # package management + dnf5-stub + rpm + createrepo_c - packages.bazel_6 = pkgsUnstable.bazel_6; + # filesystem tools + squashfsTools # mksquashfs + dosfstools # mkfs.vfat + mtools # mcopy + cryptsetup # dm-verity + util-linux # flock + kmod # depmod + cpio # cpio + zstd # zstd + xz # xz - packages.createrepo_c = pkgsUnstable.createrepo_c; + # utils + gnused # sed + gnugrep # grep + ] + ); + } + ); + in + { + # Use `legacyPackages` instead of `packages` for the reason explained here: + # https://github.com/NixOS/nixpkgs/blob/34def00657d7c45c51b0762eb5f5309689a909a5/flake.nix#L138-L156 + # Note that it's *not* a legacy attribute. + legacyPackages = { + generate = pkgs.callPackage ./nix/generate.nix { }; + } // pkgs; - packages.dnf5 = pkgsUnstable.dnf5; + packages.mkosi = mkosiDev; - devShells.default = import ./nix/shells/default.nix { pkgs = pkgsUnstable; }; + packages.uplosi = pkgs.uplosi; - formatter = nixpkgsUnstable.legacyPackages.${system}.nixpkgs-fmt; - }); + packages.openssl = callPackage ./nix/cc/openssl.nix { pkgs = pkgs; }; + + packages.cryptsetup = callPackage ./nix/cc/cryptsetup.nix { + pkgs = pkgs; + pkgsLinux = import nixpkgs { system = "x86_64-linux"; }; + }; + + packages.libvirt = callPackage ./nix/cc/libvirt.nix { + pkgs = pkgs; + pkgsLinux = import nixpkgs { system = "x86_64-linux"; }; + }; + + packages.libvirtd_base = callPackage ./nix/container/libvirtd_base.nix { + pkgs = pkgs; + pkgsLinux = import nixpkgs { system = "x86_64-linux"; }; + }; + + packages.vpn = callPackage ./nix/container/vpn/vpn.nix { + pkgs = pkgs; + pkgsLinux = import nixpkgs { system = "x86_64-linux"; }; + }; + + packages.awscli2 = pkgs.awscli2; + + packages.createrepo_c = pkgs.createrepo_c; + + packages.dnf5 = pkgs.dnf5; + + devShells.default = callPackage ./nix/shells/default.nix { }; + + formatter = nixpkgs.legacyPackages.${system}.nixpkgs-fmt; + } + ); } diff --git a/go.mod b/go.mod index 671cb6f6c..6186aca26 100644 --- a/go.mod +++ b/go.mod @@ -1,355 +1,404 @@ module github.com/edgelesssys/constellation/v2 -go 1.21 +go 1.24.4 -replace ( - k8s.io/api v0.0.0 => k8s.io/api v0.28.2 - k8s.io/apiextensions-apiserver v0.0.0 => k8s.io/apiextensions-apiserver v0.27.2 - k8s.io/apimachinery v0.0.0 => k8s.io/apimachinery v0.27.2 - k8s.io/apiserver v0.0.0 => k8s.io/apiserver v0.27.2 - k8s.io/cli-runtime v0.0.0 => k8s.io/cli-runtime v0.27.2 - k8s.io/client-go v0.0.0 => k8s.io/client-go v0.27.2 - k8s.io/cloud-provider v0.0.0 => k8s.io/cloud-provider v0.27.2 - k8s.io/cluster-bootstrap v0.0.0 => k8s.io/cluster-bootstrap v0.27.2 - k8s.io/code-generator v0.0.0 => k8s.io/code-generator v0.27.2 - k8s.io/component-base v0.0.0 => k8s.io/component-base v0.27.2 - k8s.io/component-helpers v0.0.0 => k8s.io/component-helpers v0.27.2 - k8s.io/controller-manager v0.0.0 => k8s.io/controller-manager v0.27.2 - k8s.io/cri-api v0.0.0 => k8s.io/cri-api v0.27.2 - k8s.io/csi-translation-lib v0.0.0 => k8s.io/csi-translation-lib v0.27.2 - k8s.io/dynamic-resource-allocation v0.0.0 => k8s.io/dynamic-resource-allocation v0.27.2 - k8s.io/kube-aggregator v0.0.0 => k8s.io/kube-aggregator v0.27.2 - k8s.io/kube-controller-manager v0.0.0 => k8s.io/kube-controller-manager v0.27.2 - k8s.io/kube-proxy v0.0.0 => k8s.io/kube-proxy v0.27.2 - k8s.io/kube-scheduler v0.0.0 => k8s.io/kube-scheduler v0.27.2 - k8s.io/kubectl v0.0.0 => k8s.io/kubectl v0.27.2 - k8s.io/kubelet v0.0.0 => k8s.io/kubelet v0.27.2 - k8s.io/legacy-cloud-providers v0.0.0 => k8s.io/legacy-cloud-providers v0.27.2 - k8s.io/metrics v0.0.0 => k8s.io/metrics v0.27.2 - k8s.io/mount-utils v0.0.0 => k8s.io/mount-utils v0.27.2 - k8s.io/pod-security-admission v0.0.0 => k8s.io/pod-security-admission v0.27.2 - k8s.io/sample-apiserver v0.0.0 => k8s.io/sample-apiserver v0.27.2 -) +// TODO(daniel-weisse): revert after merging https://github.com/martinjungblut/go-cryptsetup/pull/16. +replace github.com/martinjungblut/go-cryptsetup => github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c +// TODO(daniel-weisse): revert after merging https://github.com/google/go-sev-guest/pull/173. +replace github.com/google/go-sev-guest => github.com/daniel-weisse/go-sev-guest v0.0.0-20250728114912-0c2ba277c52b + +// Kubernetes replace directives are required because we depend on k8s.io/kubernetes/cmd/kubeadm +// k8s discourages usage of k8s.io/kubernetes as a dependency, but no external staging repositories for kubeadm exist. +// Our code does not actually depend on these packages, but `go mod download` breaks without this replace directive. +// See this issue: https://github.com/kubernetes/kubernetes/issues/79384 +// And this README: https://github.com/kubernetes/kubernetes/blob/master/staging/README.md replace ( - github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api => ./operators/constellation-node-operator/api - github.com/google/go-sev-guest => github.com/google/go-sev-guest v0.0.0-20230928233922-2dcbba0a4b9d - github.com/google/go-tpm => github.com/thomasten/go-tpm v0.0.0-20230629092004-f43f8e2a59eb - github.com/martinjungblut/go-cryptsetup => github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c - github.com/tink-crypto/tink-go/v2 v2.0.0 => github.com/derpsteb/tink-go/v2 v2.0.0-20231002051717-a808e454eed6 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.33.3 + k8s.io/controller-manager => k8s.io/controller-manager v0.33.3 + k8s.io/cri-client => k8s.io/cri-client v0.33.3 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.33.3 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.33.3 + k8s.io/endpointslice => k8s.io/endpointslice v0.33.3 + k8s.io/externaljwt => k8s.io/externaljwt v0.33.3 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.33.3 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.33.3 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.33.3 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.33.3 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.30.14 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.33.3 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.33.3 ) require ( - cloud.google.com/go/compute v1.23.0 - cloud.google.com/go/compute/metadata v0.2.3 - cloud.google.com/go/kms v1.15.2 - cloud.google.com/go/logging v1.8.1 - cloud.google.com/go/secretmanager v1.11.1 - cloud.google.com/go/storage v1.31.0 - dario.cat/mergo v1.0.0 + cloud.google.com/go/compute v1.41.0 + cloud.google.com/go/compute/metadata v0.7.0 + cloud.google.com/go/kms v1.22.0 + cloud.google.com/go/secretmanager v1.15.0 + cloud.google.com/go/storage v1.56.0 + dario.cat/mergo v1.0.2 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights v1.1.1 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.0.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 - github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/aws/aws-sdk-go v1.44.297 - github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 - github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.8 - github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.13 - github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.14 - github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.10 - github.com/aws/smithy-go v1.13.5 - github.com/bazelbuild/rules_go v0.42.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.4.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 + github.com/BurntSushi/toml v1.5.0 + github.com/aws/aws-sdk-go v1.55.7 + github.com/aws/aws-sdk-go-v2 v1.37.1 + github.com/aws/aws-sdk-go-v2/config v1.30.2 + github.com/aws/aws-sdk-go-v2/credentials v1.18.2 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.2 + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.55.1 + github.com/aws/aws-sdk-go-v2/service/cloudfront v1.49.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.238.0 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.47.1 + github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.27.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.85.1 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.36.1 + github.com/aws/smithy-go v1.22.5 + github.com/bazelbuild/buildtools v0.0.0-20250715102656-62b9413b08bb + github.com/bazelbuild/rules_go v0.55.1 github.com/coreos/go-systemd/v22 v22.5.0 - github.com/docker/docker v24.0.7+incompatible - github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api v0.0.0 - github.com/edgelesssys/go-azguestattestation v0.0.0-20230707101700-a683be600fcf - github.com/edgelesssys/go-tdx-qpl v0.0.0-20230530085549-fd2878a4dead - github.com/fsnotify/fsnotify v1.7.0 + github.com/docker/docker v28.2.2+incompatible + github.com/edgelesssys/go-azguestattestation v0.0.0-20250408071817-8c4457b235ff + github.com/edgelesssys/go-tdx-qpl v0.0.0-20250129202750-607ac61e2377 + github.com/foxboron/go-uefi v0.0.0-20250625111927-a3183a1bfc84 + github.com/fsnotify/fsnotify v1.9.0 github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator v0.18.1 - github.com/go-playground/validator/v10 v10.14.1 - github.com/golang-jwt/jwt/v5 v5.0.0 - github.com/google/go-sev-guest v0.9.3 - github.com/google/go-tpm v0.9.0 - github.com/google/go-tpm-tools v0.4.2 - github.com/google/uuid v1.4.0 - github.com/googleapis/gax-go/v2 v2.12.0 - github.com/gophercloud/gophercloud v1.5.0 - github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 - github.com/hashicorp/go-kms-wrapping/v2 v2.0.10 - github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.7 - github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.7 - github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.8 - github.com/hashicorp/go-version v1.6.0 - github.com/hashicorp/hc-install v0.6.1 - github.com/hashicorp/hcl/v2 v2.19.1 - github.com/hashicorp/terraform-exec v0.19.0 - github.com/hashicorp/terraform-json v0.18.0 + github.com/go-playground/validator/v10 v10.27.0 + github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/google/go-sev-guest v0.13.0 + github.com/google/go-tdx-guest v0.3.2-0.20250505161510-9efd53b4a100 + github.com/google/go-tpm v0.9.5 + github.com/google/go-tpm-tools v0.4.4 + github.com/google/uuid v1.6.0 + github.com/googleapis/gax-go/v2 v2.15.0 + github.com/gophercloud/gophercloud/v2 v2.7.0 + github.com/gophercloud/utils/v2 v2.0.0-20250711132455-9770683b100a + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 + github.com/hashicorp/go-kms-wrapping/v2 v2.0.18 + github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.11 + github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.14 + github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.13 + github.com/hashicorp/go-version v1.7.0 + github.com/hashicorp/hc-install v0.9.2 + github.com/hashicorp/hcl/v2 v2.24.0 + github.com/hashicorp/terraform-exec v0.23.0 + github.com/hashicorp/terraform-json v0.25.0 + github.com/hashicorp/terraform-plugin-framework v1.15.0 + github.com/hashicorp/terraform-plugin-framework-validators v0.18.0 + github.com/hashicorp/terraform-plugin-go v0.28.0 + github.com/hashicorp/terraform-plugin-log v0.9.0 + github.com/hashicorp/terraform-plugin-testing v1.13.2 + github.com/hexops/gotextdiff v1.0.3 github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 - github.com/mattn/go-isatty v0.0.19 - github.com/microsoft/ApplicationInsights-Go v0.4.4 + github.com/mattn/go-isatty v0.0.20 + github.com/mitchellh/go-homedir v1.1.0 + github.com/onsi/ginkgo/v2 v2.23.4 + github.com/onsi/gomega v1.38.0 github.com/pkg/errors v0.9.1 - github.com/rogpeppe/go-internal v1.11.0 - github.com/schollz/progressbar/v3 v3.13.1 - github.com/siderolabs/talos/pkg/machinery v1.4.6 - github.com/sigstore/rekor v1.2.2 - github.com/sigstore/sigstore v1.7.1 - github.com/spf13/afero v1.10.0 - github.com/spf13/cobra v1.7.0 - github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.4 - github.com/theupdateframework/go-tuf v0.5.2 - github.com/tink-crypto/tink-go/v2 v2.0.0 + github.com/regclient/regclient v0.9.0 + github.com/rogpeppe/go-internal v1.14.1 + github.com/samber/slog-multi v1.4.1 + github.com/schollz/progressbar/v3 v3.18.0 + github.com/secure-systems-lab/go-securesystemslib v0.9.0 + github.com/siderolabs/talos/pkg/machinery v1.10.5 + github.com/sigstore/rekor v1.3.10 + github.com/sigstore/sigstore v1.9.5 + github.com/spf13/afero v1.14.0 + github.com/spf13/cobra v1.9.1 + github.com/spf13/pflag v1.0.7 + github.com/stretchr/testify v1.10.0 + github.com/tink-crypto/tink-go/v2 v2.4.0 github.com/vincent-petithory/dataurl v1.0.0 + go.etcd.io/etcd/api/v3 v3.6.4 + go.etcd.io/etcd/client/pkg/v3 v3.6.4 + go.etcd.io/etcd/client/v3 v3.6.4 go.uber.org/goleak v1.3.0 - go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.14.0 - golang.org/x/mod v0.13.0 - golang.org/x/sys v0.13.0 - golang.org/x/tools v0.14.0 - google.golang.org/api v0.148.0 - google.golang.org/grpc v1.59.0 - google.golang.org/protobuf v1.31.0 + golang.org/x/crypto v0.40.0 + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 + golang.org/x/mod v0.26.0 + golang.org/x/sys v0.34.0 + golang.org/x/text v0.27.0 + golang.org/x/tools v0.35.0 + google.golang.org/api v0.244.0 + google.golang.org/grpc v1.74.2 + google.golang.org/protobuf v1.36.6 gopkg.in/yaml.v3 v3.0.1 - helm.sh/helm v2.17.0+incompatible - helm.sh/helm/v3 v3.13.1 - k8s.io/api v0.28.2 - k8s.io/apiextensions-apiserver v0.28.2 - k8s.io/apimachinery v0.28.2 - k8s.io/apiserver v0.28.2 - k8s.io/client-go v0.28.2 - k8s.io/cluster-bootstrap v0.27.3 - k8s.io/kubelet v0.27.3 - k8s.io/kubernetes v1.27.8 - k8s.io/mount-utils v0.27.3 - k8s.io/utils v0.0.0-20230505201702-9f6742963106 - sigs.k8s.io/yaml v1.3.0 + helm.sh/helm/v3 v3.18.4 + k8s.io/api v0.33.3 + k8s.io/apiextensions-apiserver v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/apiserver v0.33.3 + k8s.io/client-go v0.33.3 + k8s.io/cluster-bootstrap v0.33.3 + k8s.io/kubelet v0.33.3 + k8s.io/kubernetes v1.33.3 + k8s.io/mount-utils v0.33.3 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + libvirt.org/go/libvirt v1.11005.0 + sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/yaml v1.6.0 ) require ( - github.com/Microsoft/hcsshim v0.11.0 // indirect - github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/google/gnostic-models v0.6.8 // indirect -) - -require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/iam v1.1.2 // indirect - cloud.google.com/go/longrunning v0.5.1 // indirect - code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect - github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.4 // indirect + cloud.google.com/go/auth v0.16.3 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/longrunning v0.6.7 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect - github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/Masterminds/sprig/v3 v3.2.3 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect - github.com/agext/levenshtein v1.2.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/agext/levenshtein v1.2.2 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.26.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.35.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/cloudflare/circl v1.3.3 // indirect - github.com/containerd/containerd v1.7.6 // indirect + github.com/cloudflare/circl v1.6.1 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/containerd/containerd v1.7.27 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/coredns/caddy v1.1.1 // indirect + github.com/coredns/corefile-migration v1.0.25 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/cli v24.0.6+incompatible // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker-credential-helpers v0.7.0 // indirect - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-metrics v0.0.1 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/danieljoos/wincred v1.2.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/foxboron/go-uefi v0.0.0-20230808201820-18b9ba9cd4c3 - github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/evanphx/json-patch v5.9.11+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-jose/go-jose/v3 v3.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/analysis v0.21.4 // indirect - github.com/go-openapi/errors v0.20.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/loads v0.21.2 // indirect - github.com/go-openapi/runtime v0.26.0 // indirect - github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/strfmt v0.21.7 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-openapi/validate v0.22.1 // indirect - github.com/go-sql-driver/mysql v1.7.1 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.1 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gofrs/uuid v4.2.0+incompatible // indirect + github.com/gofrs/uuid/v5 v5.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/certificate-transparency-go v1.1.4 // indirect - github.com/google/go-attestation v0.5.0 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-containerregistry v0.15.2 // indirect - github.com/google/go-tdx-guest v0.2.3-0.20231011100059-4cf02bed9d33 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/certificate-transparency-go v1.1.8 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-attestation v0.5.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/go-configfs-tsm v0.3.3-0.20240919001351-b4b5b84fdcbc // indirect + github.com/google/go-containerregistry v0.20.3 // indirect github.com/google/go-tspi v0.3.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect github.com/google/logger v1.1.1 // indirect - github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c // indirect - github.com/google/s2a-go v0.1.7 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect - github.com/gorilla/mux v1.8.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-checkpoint v0.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-cty v1.5.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 // indirect - github.com/hashicorp/go-secure-stdlib/awsutil v0.2.2 // indirect + github.com/hashicorp/go-plugin v1.6.3 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect + github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.6 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/huandu/xstrings v1.4.0 // indirect - github.com/imdario/mergo v0.3.15 // indirect + github.com/hashicorp/logutils v1.0.0 // indirect + github.com/hashicorp/terraform-plugin-sdk/v2 v2.37.0 // indirect + github.com/hashicorp/terraform-registry-address v0.2.5 // indirect + github.com/hashicorp/terraform-svchost v0.1.1 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/huandu/xstrings v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jmoiron/sqlx v1.3.5 // indirect + github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/leodido/go-urn v1.2.4 // indirect - github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec // indirect github.com/lib/pq v1.10.9 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/lithammer/dedent v1.1.0 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/locker v1.0.1 // indirect - github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect - github.com/moby/term v0.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/oklog/run v1.0.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pborman/uuid v1.2.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/rubenv/sql-migrate v1.5.2 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20241121165744-79df5c4772f2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rubenv/sql-migrate v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/samber/lo v1.51.0 // indirect + github.com/samber/slog-common v0.19.0 // indirect github.com/sassoftware/relic v7.2.1+incompatible // indirect - github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sigstore/protobuf-specs v0.4.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/theupdateframework/go-tuf v0.7.0 // indirect github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect github.com/transparency-dev/merkle v0.0.2 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v1.2.0 // indirect - github.com/zclconf/go-cty v1.14.1 // indirect - go.mongodb.org/mongo-driver v1.11.3 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect - go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + github.com/zclconf/go-cty v1.16.3 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/otel/sdk v1.36.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect + go.opentelemetry.io/otel/trace v1.36.0 // indirect + go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 - golang.org/x/time v0.3.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.3 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/time v0.12.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250721164621-a45f3dfb1074 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/cli-runtime v0.28.2 // indirect - k8s.io/component-base v0.28.2 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/kubectl v0.28.2 // indirect - oras.land/oras-go v1.2.4 // indirect - sigs.k8s.io/controller-runtime v0.15.0 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + gotest.tools/v3 v3.4.0 // indirect + k8s.io/cli-runtime v0.33.2 // indirect + k8s.io/component-base v0.33.3 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/kubectl v0.33.2 // indirect + oras.land/oras-go/v2 v2.6.0 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/kustomize/api v0.19.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.19.0 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect ) diff --git a/go.sum b/go.sum index 07d383cee..a974116aa 100644 --- a/go.sum +++ b/go.sum @@ -1,106 +1,78 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= -cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/kms v1.15.2 h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE= -cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= -cloud.google.com/go/logging v1.8.1 h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU= -cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= -cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs= -cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= -cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= -code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ= -code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.4 h1:cVvUiY0sX0xwyxPwdSU2KsF9knOVmtRyAMt8xou0iTs= +cloud.google.com/go v0.121.4/go.mod h1:XEBchUiHFJbz4lKBZwYBDHV/rSyfFktk737TLDU089s= +cloud.google.com/go/auth v0.16.3 h1:kabzoQ9/bobUmnseYnBO6qQG7q4a/CffFRlJSxv2wCc= +cloud.google.com/go/auth v0.16.3/go.mod h1:NucRGjaXfzP1ltpcQ7On/VTZ0H4kWB5Jy+Y9Dnm76fA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute v1.41.0 h1:S+HvMIzBUAFK/73wxkrA4/GwvM7R9d+egGZvih4kp+M= +cloud.google.com/go/compute v1.41.0/go.mod h1:P1doTJnlwurJDzIQFMp4mgU+vyCe9HU2NWTlqTfq3MY= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/kms v1.22.0 h1:dBRIj7+GDeeEvatJeTB19oYZNV0aj6wEqSIT/7gLqtk= +cloud.google.com/go/kms v1.22.0/go.mod h1:U7mf8Sva5jpOb4bxYZdtw/9zsbIjrklYwPcvMk34AL8= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/secretmanager v1.15.0 h1:RtkCMgTpaBMbzozcRUGfZe46jb9a3qh5EdEtVRUATF8= +cloud.google.com/go/secretmanager v1.15.0/go.mod h1:1hQSAhKK7FldiYw//wbR/XPfPc08eQ81oBsnRUHEvUc= +cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= +cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230618160516-e936619f9f18 h1:rd389Q26LMy03gG4anandGFC2LW/xvjga5GezeeaxQk= -github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230618160516-e936619f9f18/go.mod h1:fgJuSBrJP5qZtKqaMJE0hmhS2tmRH+44IkfZvjtaf1M= +github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg= +github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d/go.mod h1:XNqJ7hv2kY++g8XEHREpi+JqZo3+0l+CH2egBVN4yqM= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= -github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights v1.1.1 h1:hBrFatNIiVAwDb5GzMLjpkQ6l2/waFSvBWMBWZRH8WI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights v1.1.1/go.mod h1:uxknLoFj+nBXpfGngz0B4ciNur04Y0EX4AREpy2GIvk= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 h1:Sg/D8VuUQ+bw+FOYJF+xRKcwizCOP13HL0Se8pWNBzE= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0/go.mod h1:Kyqzdqq0XDoCm+o9aZ25wZBmBUBzPBzPAj1R5rYsT6I= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2/go.mod h1:FbdwsQ2EzwvXxOPcMFYO8ogEc9uMMIj3YkmCdXdAFmk= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.0.0 h1:pqCyNi/Paz03SbWRmGlb5WBzK14aOXVuSJuOTWzOM5M= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.0.0/go.mod h1:bCUhQ1sbQHAG4nm1SqWwLlnKnRVT2e6Lu0cij7OzliM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.4.0 h1:z7Mqz6l0EFH549GvHEqfjKvi+cRScxLWbaoeLm9wxVQ= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.4.0/go.mod h1:v6gbfH+7DG7xH2kUNs+ZJ9tF6O3iNnR85wMtmr+F54o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 h1:HYGD75g0bQ3VO/Omedm54v4LrD3B1cGImuRF3AJ5wLo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0/go.mod h1:ulHyBFJOI0ONiRL4vcJTmS7rx18jQQlEPmAgo80cRdM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= @@ -114,107 +86,104 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= -github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= -github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= -github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= +github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.297 h1:uL4EV0gQxotQVYegIoBqK079328MOJqgG95daFYSkAM= -github.com/aws/aws-sdk-go v1.44.297/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= -github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.8 h1:loRDtQ0vT0+JCB0hQBCfv95tttEzJ1rqSaTDy5cpy0A= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.8/go.mod h1:YTd4wGn2beCF9wkSTpEcupk79zDFYJk2Ca76B8YyvJg= -github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2 h1:zMnh9plMceN5DVuG55IjzEwAS3kbeG0GTNzmbnqI/C8= -github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2/go.mod h1:zuZWQM3kYD+ibkP3GBrAMbsfUvHK7p7yOwWh9MKsnYQ= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 h1:P4dyjm49F2kKws0FpouBC6fjVImACXKt752+CWa01lM= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0/go.mod h1:tIctCeX9IbzsUTKHt53SVEcgyfxV2ElxJeEB+QUbc4M= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.13 h1:g/Kzed9qNdvz5p7Av3ffavD19eN11deWqlHgR2JuXuw= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.19.13/go.mod h1:BNkuX97Xp8meRKwZkWlXajo3u4cP/B3TC+YsadbOfaM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.14 h1:6AuIiaZ+oRhprPZw2/siZQcaZRvmKipjGbmGI0BSGsA= -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.14.14/go.mod h1:w4zYOP9Y8sYBattA+ysl0tDy72+aO6a3d6FrkK2FgZc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.10 h1:eW8zPSh7ZLzb7029xCsIEFbnxLvNHPTt7aWwdKjNJc8= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.10/go.mod h1:ezn6mzIRqTPdAbDpm03dx4y9g6rvGRb2q33wS76dCxw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/bazelbuild/rules_go v0.42.0 h1:aY2smc3JWyUKOjGYmOKVLX70fPK9ON0rtwQojuIeUHc= -github.com/bazelbuild/rules_go v0.42.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= +github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.37.1 h1:SMUxeNz3Z6nqGsXv0JuJXc8w5YMtrQMuIBmDx//bBDY= +github.com/aws/aws-sdk-go-v2 v1.37.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= +github.com/aws/aws-sdk-go-v2/config v1.30.2 h1:YE1BmSc4fFYqFgN1mN8uzrtc7R9x+7oSWeX8ckoltAw= +github.com/aws/aws-sdk-go-v2/config v1.30.2/go.mod h1:UNrLGZ6jfAVjgVJpkIxjLufRJqTXCVYOpkeVf83kwBo= +github.com/aws/aws-sdk-go-v2/credentials v1.18.2 h1:mfm0GKY/PHLhs7KO0sUaOtFnIQ15Qqxt+wXbO/5fIfs= +github.com/aws/aws-sdk-go-v2/credentials v1.18.2/go.mod h1:v0SdJX6ayPeZFQxgXUKw5RhLpAoZUuynxWDfh8+Eknc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.1 h1:owmNBboeA0kHKDcdF8KiSXmrIuXZustfMGGytv6OMkM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.1/go.mod h1:Bg1miN59SGxrZqlP8vJZSmXW+1N8Y1MjQDq1OfuNod8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.2 h1:YFX4DvH1CPQXgQR8935b46Om+L7+6jus4aTdKqyDR84= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.2/go.mod h1:DgMPy7GqxcV0RSyaITnI3rw8HC3lIHB87U3KPQKDxHg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.1 h1:ksZXBYv80EFTcgc8OJO48aQ8XDWXIQL7gGasPeCoTzI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.1/go.mod h1:HSksQyyJETVZS7uM54cir0IgxttTD+8aEoJMPGepHBI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.1 h1:+dn/xF/05utS7tUhjIcndbuaPjfll2LhbH1cCDGLYUQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.1/go.mod h1:hyAGz30LHdm5KBZDI58MXx5lDVZ5CUfvfTZvMu4HCZo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.1 h1:4HbnOGE9491a9zYJ9VpPh1ApgEq6ZlD4Kuv1PJenFpc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.1/go.mod h1:Z6QnHC6TmpJWUxAy8FI4JzA7rTwl6EIANkyK9OR5z5w= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.55.1 h1:zX6/huIuV5ldMXSiVVdmRT2oO1M+xNLzdt0du0QuhVE= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.55.1/go.mod h1:KWk5jIp+F7eu9vjz6g/UdeIk5FX2zw7zllkf8EwmHjM= +github.com/aws/aws-sdk-go-v2/service/cloudfront v1.49.0 h1:ZABkPLtfK+q2GkW1pA+NukaGM/EAKamEUR347B1md2U= +github.com/aws/aws-sdk-go-v2/service/cloudfront v1.49.0/go.mod h1:PHC5ybfgglvCqD7fLaqR5A7LIuJqIoUxhlwF/8faMt0= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.238.0 h1:fXZYx7xDSocFM3ht/mwML7eCP7cPbs1ltXEM8zpwU5o= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.238.0/go.mod h1:lhyI/MJGGbPnOdYmmQRZe07S+2fW2uWI1XrUfAZgXLM= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.47.1 h1:H8+KiNkkY3q3u7IUSjc7oCshnHOOGvYOi7fT6ZJ23OI= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.47.1/go.mod h1:91PY/MUWThH0rH61v9r3QA4e7dS/PfXl+K63wltBeas= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.1 h1:ps3nrmBWdWwakZBydGX1CxeYFK80HsQ79JLMwm7Y4/c= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.1/go.mod h1:bAdfrfxENre68Hh2swNaGEVuFYE74o0SaSCAlaG9E74= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.1 h1:ky79ysLMxhwk5rxJtS+ILd3Mc8kC5fhsLBrP27r6h4I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.1/go.mod h1:+2MmkvFvPYM1vsozBWduoLJUi5maxFk5B7KJFECujhY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.1 h1:MdVYlN5pcQu1t1OYx4Ajo3fKl1IEhzgdPQbYFCRjYS8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.1/go.mod h1:iikmNLrvHm2p4a3/4BPeix2S9P+nW8yM1IZW73x8bFA= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.27.1 h1:dGw/U6NbhnWoW2gw+75/AZvnYjFuxYRtzUpxALoRhLc= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.27.1/go.mod h1:ZNIISn1QONFDUbTmkIK53IBTrGn1TbsrBH5pG/BCwew= +github.com/aws/aws-sdk-go-v2/service/s3 v1.85.1 h1:Hsqo8+dFxSdDvv9B2PgIx1AJAnDpqgS0znVI+R+MoGY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.85.1/go.mod h1:8Q0TAPXD68Z8YqlcIGHs/UNIDHsxErV9H4dl4vJEpgw= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.36.1 h1:fnOIjzwTVrtVnkRef3Qs+uTr3qYKwXuFom5pqdZERNQ= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.36.1/go.mod h1:/19D53IxSX9W8uu5bo0t89oCLncvNP68V1KiRthhLd4= +github.com/aws/aws-sdk-go-v2/service/sso v1.26.1 h1:uWaz3DoNK9MNhm7i6UGxqufwu3BEuJZm72WlpGwyVtY= +github.com/aws/aws-sdk-go-v2/service/sso v1.26.1/go.mod h1:ILpVNjL0BO+Z3Mm0SbEeUoYS9e0eJWV1BxNppp0fcb8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.1 h1:XdG6/o1/ZDmn3wJU5SRAejHaWgKS4zHv0jBamuKuS2k= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.31.1/go.mod h1:oiotGTKadCOCl3vg/tYh4k45JlDF81Ka8rdumNhEnIQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.35.1 h1:iF4Xxkc0H9c/K2dS0zZw3SCkj0Z7n6AMnUiiyoJND+I= +github.com/aws/aws-sdk-go-v2/service/sts v1.35.1/go.mod h1:0bxIatfN0aLq4mjoLDeBpOjOke68OsFlXPDFJ7V0MYw= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/bazelbuild/buildtools v0.0.0-20250715102656-62b9413b08bb h1:KCEfAAZ5hZH+CEMzShOZ4nCro20Ohf6whugQc2PLZiM= +github.com/bazelbuild/buildtools v0.0.0-20250715102656-62b9413b08bb/go.mod h1:PLNUetjLa77TCCziPsz0EI8a6CUxgC+1jgmWv0H25tg= +github.com/bazelbuild/rules_go v0.55.1 h1:cQYGcunY8myOB+0Ym6PGQRhc/milkRcNv0my3XgxaDU= +github.com/bazelbuild/rules_go v0.55.1/go.mod h1:T90Gpyq4HDFlsrvtQa2CBdHNJ2P4rAu/uUTmQbanzf0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -223,310 +192,223 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY= +github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= -github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= -github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= -github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= +github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0= +github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= +github.com/coredns/corefile-migration v1.0.25 h1:/XexFhM8FFlFLTS/zKNEWgIZ8Gl5GaWrHsMarGj/PRQ= +github.com/coredns/corefile-migration v1.0.25/go.mod h1:56DPqONc3njpVPsdilEnfijCwNGC3/kTJLl7i7SPavY= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 h1:vU+EP9ZuFUCYE0NYLwTSob+3LNEJATzNfP/DC7SWGWI= github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c h1:ToajP6trZoiqlZ3Z4uoG1P02/wtqSw1AcowOXOYjATk= github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c/go.mod h1:gZoZ0+POlM1ge/VUxWpMmZVNPzzMJ7l436CgkQ5+qzU= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/daniel-weisse/go-sev-guest v0.0.0-20250728114912-0c2ba277c52b h1:pElX9BS0PnYZS/tznradDYbo82kvG2yisWGvZGsDnVs= +github.com/daniel-weisse/go-sev-guest v0.0.0-20250728114912-0c2ba277c52b/go.mod h1:SK9vW+uyfuzYdVN0m8BShL3OQCtXZe/JPF7ZkpD3760= +github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= +github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/derpsteb/tink-go/v2 v2.0.0-20231002051717-a808e454eed6 h1:FVii9oXvddz9sFir5TRYjQKrzJLbVD/hibT+SnRSDzg= -github.com/derpsteb/tink-go/v2 v2.0.0-20231002051717-a808e454eed6/go.mod h1:QAbyq9LZncomYnScxlfaHImbV4ieNIe6bnu/Xcqqox4= -github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= -github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= -github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/distribution/v3 v3.0.0 h1:q4R8wemdRQDClzoNNStftB2ZAfqOiN6UX90KJc4HjyM= +github.com/distribution/distribution/v3 v3.0.0/go.mod h1:tRNuFoZsUdyRVegq8xGNeds4KLjwLCRin/tTo6i1DhU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= +github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/edgelesssys/go-azguestattestation v0.0.0-20230707101700-a683be600fcf h1:1iKB7b+i7svWC0aKXwggi+kHf0K57g8r9hN4VOpJYYg= -github.com/edgelesssys/go-azguestattestation v0.0.0-20230707101700-a683be600fcf/go.mod h1:T8Rv3qrCpUJZbKq49OA9tcC1ZbRkGtDxiafsj++LYIE= -github.com/edgelesssys/go-tdx-qpl v0.0.0-20230530085549-fd2878a4dead h1:Q2TI34V/NCLGQQkdc0/KmPx/7ix9YnGDUQDT+gqvDw0= -github.com/edgelesssys/go-tdx-qpl v0.0.0-20230530085549-fd2878a4dead/go.mod h1:IC72qyykUIWl0ZmSk53L4xbLCFDBEGZVaujUmPQOEyw= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/edgelesssys/go-azguestattestation v0.0.0-20250408071817-8c4457b235ff h1:V6A5kD0+c1Qg4X72Lg+zxhCZk+par436sQdgLvMCBBc= +github.com/edgelesssys/go-azguestattestation v0.0.0-20250408071817-8c4457b235ff/go.mod h1:Lz4QaomI4wU2YbatD4/W7vatW2Q35tnkoJezB1clscc= +github.com/edgelesssys/go-tdx-qpl v0.0.0-20250129202750-607ac61e2377 h1:5JMJiBhvOUUR7EZ0UyeSy7a1WrqB2eM+DX3odLSHAh4= +github.com/edgelesssys/go-tdx-qpl v0.0.0-20250129202750-607ac61e2377/go.mod h1:IC72qyykUIWl0ZmSk53L4xbLCFDBEGZVaujUmPQOEyw= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= -github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg= -github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01/go.mod h1:ypD5nozFk9vcGw1ATYefw6jHe/jZP++Z15/+VTMcWhc= -github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8= -github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/foxboron/go-uefi v0.0.0-20230808201820-18b9ba9cd4c3 h1:SJMQFT74bCrP+kQ24oWhmuyPFHDTavrd3JMIe//2NhU= -github.com/foxboron/go-uefi v0.0.0-20230808201820-18b9ba9cd4c3/go.mod h1:VdozURTQHi5Rs54l+4Szi3yIJQDMfXXYrRLAjKKowWI= -github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= -github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/foxboron/go-uefi v0.0.0-20250625111927-a3183a1bfc84 h1:8VH593jGTaWbdMdkVr91w0fnzmFmgV69J2OCOjokqp8= +github.com/foxboron/go-uefi v0.0.0-20250625111927-a3183a1bfc84/go.mod h1:q85c4IRlhhwdRJgGIUWrisDjU8dgcMj8dnXZCXo3hus= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= -github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git/v5 v5.14.0 h1:/MD3lCrGjCen5WfEAzKg00MJJffKhC8gzS80ycmCi60= +github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= -github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= -github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= +github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/analysis v0.21.4 h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc= -github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.4 h1:unTcVm6PispJsMECE3zWgvG4xTiKda1LIR5rCRWLG6M= -github.com/go-openapi/errors v0.20.4/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro= -github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= -github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= -github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= -github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= -github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= -github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.1 h1:kslMRRnK7NCb/CvR1q1VWuEQCEIsBGn5GgKD9e+HYhU= +github.com/go-openapi/errors v0.22.1/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= -github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= -github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= -github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0= -github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= -github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI= +github.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= -github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gofrs/uuid/v5 v5.3.2 h1:2jfO8j3XgSwlz/wHqemAEugfnTlikAYHhnqQ8Xh4fE0= +github.com/gofrs/uuid/v5 v5.3.2/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= -github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= -github.com/google/certificate-transparency-go v1.1.4 h1:hCyXHDbtqlr/lMXU0D4WgbalXL0Zk4dSWWMbPV8VrqY= -github.com/google/certificate-transparency-go v1.1.4/go.mod h1:D6lvbfwckhNrbM9WVl1EVeMOyzC19mpIjMOI4nxBHtQ= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-attestation v0.5.0 h1:jXtAWT2sw2Yu8mYU0BC7FDidR+ngxFPSE+pl6IUu3/0= -github.com/google/go-attestation v0.5.0/go.mod h1:0Tik9y3rzV649Jcr7evbljQHQAsIlJucyqQjYDBqktU= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/certificate-transparency-go v1.1.8 h1:LGYKkgZF7satzgTak9R4yzfJXEeYVAjV6/EAEJOf1to= +github.com/google/certificate-transparency-go v1.1.8/go.mod h1:bV/o8r0TBKRf1X//iiiSgWrvII4d7/8OiA+3vG26gI8= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= +github.com/google/go-attestation v0.5.1 h1:jqtOrLk5MNdliTKjPbIPrAaRKJaKW+0LIU2n/brJYms= +github.com/google/go-attestation v0.5.1/go.mod h1:KqGatdUhg5kPFkokyzSBDxwSCFyRgIgtRkMp6c3lOBQ= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= -github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= -github.com/google/go-sev-guest v0.0.0-20230928233922-2dcbba0a4b9d h1:6o4Z/vQqNUH+cEagfx1Ez5ElK70iZulEXZwmLnRo44I= -github.com/google/go-sev-guest v0.0.0-20230928233922-2dcbba0a4b9d/go.mod h1:hc1R4R6f8+NcJwITs0L90fYWTsBpd1Ix+Gur15sqHDs= -github.com/google/go-tdx-guest v0.2.3-0.20231011100059-4cf02bed9d33 h1:lRlUusuieEuqljjihCXb+Mr73VNitOYPJYWXzJKtBWs= -github.com/google/go-tdx-guest v0.2.3-0.20231011100059-4cf02bed9d33/go.mod h1:84ut3oago/BqPXD4ppiGXdkZNW3WFPkcyAO4my2hXdY= -github.com/google/go-tpm-tools v0.4.2 h1:iyaCPKt2N5Rd0yz0G8ANa022SgCNZkMpp+db6QELtvI= -github.com/google/go-tpm-tools v0.4.2/go.mod h1:fGUDZu4tw3V4hUVuFHmiYgRd0c58/IXivn9v3Ea/ck4= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-configfs-tsm v0.3.3-0.20240919001351-b4b5b84fdcbc h1:SG12DWUUM5igxm+//YX5Yq4vhdoRnOG9HkCodkOn+YU= +github.com/google/go-configfs-tsm v0.3.3-0.20240919001351-b4b5b84fdcbc/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo= +github.com/google/go-containerregistry v0.20.3 h1:oNx7IdTI936V8CQRveCjaxOiegWwvM7kqkbXTpyiovI= +github.com/google/go-containerregistry v0.20.3/go.mod h1:w00pIgBRDVUDFM6bq+Qx8lwNWK+cxgCuX1vd3PIBDNI= +github.com/google/go-tdx-guest v0.3.2-0.20250505161510-9efd53b4a100 h1:E5ArM6vmtaUbgTZM8W3G+bgseO3i5l8BpJCKuUH7WVc= +github.com/google/go-tdx-guest v0.3.2-0.20250505161510-9efd53b4a100/go.mod h1:uHy3VaNXNXhl0fiPxKqTxieeouqQmW6A0EfLcaeCYBk= +github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= +github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm-tools v0.4.4 h1:oiQfAIkc6xTy9Fl5NKTeTJkBTlXdHsxAofmQyxBKY98= +github.com/google/go-tpm-tools v0.4.4/go.mod h1:T8jXkp2s+eltnCDIsXR84/MTcVU9Ja7bh3Mit0pa4AY= github.com/google/go-tspi v0.3.0 h1:ADtq8RKfP+jrTyIWIZDIYcKOMecRqNJFOew2IT0Inus= github.com/google/go-tspi v0.3.0/go.mod h1:xfMGI3G0PhxCdNVcYr1C4C+EizojDg/TXuX5by8CiHI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -534,154 +416,147 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ= github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gophercloud/gophercloud v1.3.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= -github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= -github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= -github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 h1:sH7xkTfYzxIEgzq1tDHIMKRh1vThOEOGNsettdEeLbE= -github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56/go.mod h1:VSalo4adEk+3sNkmVJLnhHoOyOYYS8sTWLG4mv5BKto= -github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= -github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/gophercloud/gophercloud/v2 v2.7.0 h1:o0m4kgVcPgHlcXiWAjoVxGd8QCmvM5VU+YM71pFbn0E= +github.com/gophercloud/gophercloud/v2 v2.7.0/go.mod h1:Ki/ILhYZr/5EPebrPL9Ej+tUg4lqx71/YH2JWVeU+Qk= +github.com/gophercloud/utils/v2 v2.0.0-20250711132455-9770683b100a h1:erVLycqmezd0+eukgQ4xgLxGsByDKvqJxLXVc35tUYI= +github.com/gophercloud/utils/v2 v2.0.0-20250711132455-9770683b100a/go.mod h1:1mckc18GQSFLRhDy2BjPGkkpbrjxY5iwX/oxpdTE2kw= +github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= +github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 h1:sGm2vDRFUrQJO/Veii4h4zG2vvqG6uWNkBHSTqXOZk0= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2/go.mod h1:wd1YpapPLivG6nQgbf7ZkG1hhSOXDhhn4MLTknx2aAc= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= +github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-kms-wrapping/v2 v2.0.10 h1:A51EguZ576URdtcQ0l8mT/tOD948oAtmP1soqIHIFfI= -github.com/hashicorp/go-kms-wrapping/v2 v2.0.10/go.mod h1:NtMaPhqSlfQ72XWDD2g80o8HI8RKkowIB8/WZHMyPY4= -github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.7 h1:E3eEWpkofgPNrYyYznfS1+drq4/jFcqHQVNcL7WhUCo= -github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.7/go.mod h1:j5vefRoguQUG7iM4reS/hKIZssU1lZRqNPM5Wow6UnM= -github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.7 h1:X27JWuPW6Gmi2l7NMm0pvnp7z7hhtns2TeIOQU93mqI= -github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.7/go.mod h1:i7Dt9mDsVUQG/I639jtdQerliaO2SvvPnpYPhZ8CGZ4= -github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.8 h1:16I8OqBEuxZIowwn3jiLvhlx+z+ia4dJc9stvz0yUBU= -github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.8/go.mod h1:6QUMo5BrXAtbzSuZilqmx0A4px2u6PeFK7vfp2WIzeM= +github.com/hashicorp/go-cty v1.5.0 h1:EkQ/v+dDNUqnuVpmS5fPqyY71NXVgT5gf32+57xY8g0= +github.com/hashicorp/go-cty v1.5.0/go.mod h1:lFUCG5kd8exDobgSfyj4ONE/dc822kiYMguVKdHGMLM= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-kms-wrapping/v2 v2.0.18 h1:DLfC677GfKEpSAFpEWvl1vXsGpEcSHmbhBaPLrdDQHc= +github.com/hashicorp/go-kms-wrapping/v2 v2.0.18/go.mod h1:t/eaR/mi2mw3klfl1WEAuiLKrlZ/Q8cosmsT+RIPLu0= +github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.11 h1:J9zGa9SlcOHT3SQTj0Vv3shHo0anWbs58weURGCgChI= +github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.11/go.mod h1:iAOCu7/lG5eugg8+k7NVvQt0IpWT8s2Q9wnMtC/guM4= +github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.14 h1:oK4OQ5EPbx/66dAvitksV+OdrQ86SZEj3B6VSZrbdEY= +github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.14/go.mod h1:fWxrv9YkAMqtsISde5mcutoMvuiH4kyg1AlDzzmqRh8= +github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.13 h1:NGBZnF+yPRZ3gjFl69Y2m58/U0iyB2oH9HaznL9tekA= +github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.13/go.mod h1:4Xb+6d8VPeDcUNuh4toPqJlDpkajeJyIQeg36TtWhKw= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-secure-stdlib/awsutil v0.2.2 h1:kWg2vyKl7BRXrNxYziqDJ55n+vtOQ1QsGORjzoeB+uM= -github.com/hashicorp/go-secure-stdlib/awsutil v0.2.2/go.mod h1:oKHSQs4ivIfZ3fbXGQOop1XuDfdSb8RIsWTGaAanSfg= +github.com/hashicorp/go-plugin v1.6.3 h1:xgHB+ZUSYeuJi96WtxEjzi23uh7YQpznjGh0U0UUrwg= +github.com/hashicorp/go-plugin v1.6.3/go.mod h1:MRobyh+Wc/nYy1V4KAXUiYfzxoYhs7V1mlH1Z7iY2h0= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 h1:W9WN8p6moV1fjKLkeqEgkAMu5rauy9QeYDAmIaPuuiA= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6/go.mod h1:MpCPSPGLDILGb4JMm94/mMi3YysIqsXzGCzkEZjcjXg= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9 h1:FW0YttEnUNDJ2WL9XcrrfteS1xW8u+sh4ggM8pN5isQ= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.6 h1:RSG8rKU28VTUTvEKghe5gIhIQpv8evvNpnDEyqO4u9I= +github.com/hashicorp/go-sockaddr v1.0.6/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hc-install v0.6.1 h1:IGxShH7AVhPaSuSJpKtVi/EFORNjO+OYVJJrAtGG2mY= -github.com/hashicorp/hc-install v0.6.1/go.mod h1:0fW3jpg+wraYSnFDJ6Rlie3RvLf1bIqVIkzoon4KoVE= -github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI= -github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= -github.com/hashicorp/terraform-exec v0.19.0 h1:FpqZ6n50Tk95mItTSS9BjeOVUb4eg81SpgVtZNNtFSM= -github.com/hashicorp/terraform-exec v0.19.0/go.mod h1:tbxUpe3JKruE9Cuf65mycSIT8KiNPZ0FkuTE3H4urQg= -github.com/hashicorp/terraform-json v0.18.0 h1:pCjgJEqqDESv4y0Tzdqfxr/edOIGkjs8keY42xfNBwU= -github.com/hashicorp/terraform-json v0.18.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk= -github.com/honeycombio/beeline-go v1.10.0 h1:cUDe555oqvw8oD76BQJ8alk7FP0JZ/M/zXpNvOEDLDc= -github.com/honeycombio/beeline-go v1.10.0/go.mod h1:Zz5WMeQCJzFt2Mvf8t6HC1X8RLskLVR/e8rvcmXB1G8= -github.com/honeycombio/libhoney-go v1.16.0 h1:kPpqoz6vbOzgp7jC6SR7SkNj7rua7rgxvznI6M3KdHc= -github.com/honeycombio/libhoney-go v1.16.0/go.mod h1:izP4fbREuZ3vqC4HlCAmPrcPT9gxyxejRjGtCYpmBn0= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= +github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hc-install v0.9.2 h1:v80EtNX4fCVHqzL9Lg/2xkp62bbvQMnvPQ0G+OmtO24= +github.com/hashicorp/hc-install v0.9.2/go.mod h1:XUqBQNnuT4RsxoxiM9ZaUk0NX8hi2h+Lb6/c0OZnC/I= +github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= +github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/terraform-exec v0.23.0 h1:MUiBM1s0CNlRFsCLJuM5wXZrzA3MnPYEsiXmzATMW/I= +github.com/hashicorp/terraform-exec v0.23.0/go.mod h1:mA+qnx1R8eePycfwKkCRk3Wy65mwInvlpAeOwmA7vlY= +github.com/hashicorp/terraform-json v0.25.0 h1:rmNqc/CIfcWawGiwXmRuiXJKEiJu1ntGoxseG1hLhoQ= +github.com/hashicorp/terraform-json v0.25.0/go.mod h1:sMKS8fiRDX4rVlR6EJUMudg1WcanxCMoWwTLkgZP/vc= +github.com/hashicorp/terraform-plugin-framework v1.15.0 h1:LQ2rsOfmDLxcn5EeIwdXFtr03FVsNktbbBci8cOKdb4= +github.com/hashicorp/terraform-plugin-framework v1.15.0/go.mod h1:hxrNI/GY32KPISpWqlCoTLM9JZsGH3CyYlir09bD/fI= +github.com/hashicorp/terraform-plugin-framework-validators v0.18.0 h1:OQnlOt98ua//rCw+QhBbSqfW3QbwtVrcdWeQN5gI3Hw= +github.com/hashicorp/terraform-plugin-framework-validators v0.18.0/go.mod h1:lZvZvagw5hsJwuY7mAY6KUz45/U6fiDR0CzQAwWD0CA= +github.com/hashicorp/terraform-plugin-go v0.28.0 h1:zJmu2UDwhVN0J+J20RE5huiF3XXlTYVIleaevHZgKPA= +github.com/hashicorp/terraform-plugin-go v0.28.0/go.mod h1:FDa2Bb3uumkTGSkTFpWSOwWJDwA7bf3vdP3ltLDTH6o= +github.com/hashicorp/terraform-plugin-log v0.9.0 h1:i7hOA+vdAItN1/7UrfBqBwvYPQ9TFvymaRGZED3FCV0= +github.com/hashicorp/terraform-plugin-log v0.9.0/go.mod h1:rKL8egZQ/eXSyDqzLUuwUYLVdlYeamldAHSxjUFADow= +github.com/hashicorp/terraform-plugin-sdk/v2 v2.37.0 h1:NFPMacTrY/IdcIcnUB+7hsore1ZaRWU9cnB6jFoBnIM= +github.com/hashicorp/terraform-plugin-sdk/v2 v2.37.0/go.mod h1:QYmYnLfsosrxjCnGY1p9c7Zj6n9thnEE+7RObeYs3fA= +github.com/hashicorp/terraform-plugin-testing v1.13.2 h1:mSotG4Odl020vRjIenA3rggwo6Kg6XCKIwtRhYgp+/M= +github.com/hashicorp/terraform-plugin-testing v1.13.2/go.mod h1:WHQ9FDdiLoneey2/QHpGM/6SAYf4A7AZazVg7230pLE= +github.com/hashicorp/terraform-registry-address v0.2.5 h1:2GTftHqmUhVOeuu9CW3kwDkRe4pcBDq0uuK5VJngU1M= +github.com/hashicorp/terraform-registry-address v0.2.5/go.mod h1:PpzXWINwB5kuVS5CA7m1+eO2f1jKb5ZDIxrOPfpnGkg= +github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= +github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= -github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8= github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b/go.mod h1:hQmNrgofl+IY/8L+n20H6E6PWBBTokdsv+q49j0QhsU= +github.com/jhump/protoreflect v1.15.3 h1:6SFRuqU45u9hIZPJAoZ8c28T3nK64BNdp9w6jFonzls= +github.com/jhump/protoreflect v1.15.3/go.mod h1:4ORHmSBmlCW8fh3xHmJMGyul1zNqZK4Elxc8qKP+p1k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548 h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4= -github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= +github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= -github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -695,242 +570,221 @@ github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf h1:ndns1qx/5dL43g16EQkPV/i8+b3l5bYQwLeoSBe7tS8= -github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf/go.mod h1:aGkAgvWY/IUcVFfuly53REpfv5edu25oij+qHRFaraA= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec h1:2tTW6cDth2TSgRbAhD7yjZzTQmcN25sDRPEeinR51yQ= +github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec/go.mod h1:TmwEoGCwIti7BCeJ9hescZgRtatxRE+A72pCoPfmcfk= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= -github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= -github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= -github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= +github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/olareg/olareg v0.1.2 h1:75G8X6E9FUlzL/CSjgFcYfMgNzlc7CxULpUUNsZBIvI= +github.com/olareg/olareg v0.1.2/go.mod h1:TWs+N6pO1S4bdB6eerzUm/ITRQ6kw91mVf9ZYeGtw+Y= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= +github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= -github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/planetscale/vtprotobuf v0.6.1-0.20241121165744-79df5c4772f2 h1:1sLMdKq4gNANTj0dUibycTLzpIEKVnLnbaEkxws78nw= +github.com/planetscale/vtprotobuf v0.6.1-0.20241121165744-79df5c4772f2/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= +github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= +github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= +github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ= +github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= +github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/regclient/regclient v0.9.0 h1:c3hNJZvtv8lMqhP0jGCa4d9j2n4688VCfhCWddGfWfk= +github.com/regclient/regclient v0.9.0/go.mod h1:Adv7tukwdX+oDTszfILrjerGk55Pg2nKlbshj94U3rg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= -github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rubenv/sql-migrate v1.8.0 h1:dXnYiJk9k3wetp7GfQbKJcPHjVJL6YK19tKj8t2Ns0o= +github.com/rubenv/sql-migrate v1.8.0/go.mod h1:F2bGFBwCU+pnmbtNYDeKvSuvL6lBVtXDXUUv5t+u1qw= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= +github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= +github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI= +github.com/samber/slog-common v0.19.0/go.mod h1:dTz+YOU76aH007YUU0DffsXNsGFQRQllPQh9XyNoA3M= +github.com/samber/slog-multi v1.4.1 h1:OVBxOKcorBcGQVKjwlraA41JKWwHQyB/3KfzL3IJAYg= +github.com/samber/slog-multi v1.4.1/go.mod h1:im2Zi3mH/ivSY5XDj6LFcKToRIWPw1OcjSVSdXt+2d0= github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk= -github.com/sassoftware/relic/v7 v7.5.5 h1:2ZUM6ovo3STCAp0hZnO9nQY9lOB8OyfneeYIi4YUxMU= -github.com/sassoftware/relic/v7 v7.5.5/go.mod h1:NxwtWxWxlUa9as2qZi635Ye6bBT/tGnMALLq7dSfOOU= -github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= -github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= -github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA= -github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/siderolabs/talos/pkg/machinery v1.4.6 h1:SX7Q6FxTDyX2hxugMgIqyivXWzemgMhHj3AlDbxjuFw= -github.com/siderolabs/talos/pkg/machinery v1.4.6/go.mod h1:l+3Akw0tz/k/IMbqQS+cdQaqVSitIIJf04+N9ahGn+E= -github.com/sigstore/rekor v1.2.2 h1:5JK/zKZvcQpL/jBmHvmFj3YbpDMBQnJQ6ygp8xdF3bY= -github.com/sigstore/rekor v1.2.2/go.mod h1:FGnWBGWzeNceJnp0x9eDFd41mI8aQqCjj+Zp0IEs0Qg= -github.com/sigstore/sigstore v1.7.1 h1:fCATemikcBK0cG4+NcM940MfoIgmioY1vC6E66hXxks= -github.com/sigstore/sigstore v1.7.1/go.mod h1:0PmMzfJP2Y9+lugD0wer4e7TihR5tM7NcIs3bQNk5xg= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sassoftware/relic/v7 v7.6.2 h1:rS44Lbv9G9eXsukknS4mSjIAuuX+lMq/FnStgmZlUv4= +github.com/sassoftware/relic/v7 v7.6.2/go.mod h1:kjmP0IBVkJZ6gXeAu35/KCEfca//+PKM6vTAsyDPY+k= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/secure-systems-lab/go-securesystemslib v0.9.0 h1:rf1HIbL64nUpEIZnjLZ3mcNEL9NBPB0iuVjyxvq3LZc= +github.com/secure-systems-lab/go-securesystemslib v0.9.0/go.mod h1:DVHKMcZ+V4/woA/peqr+L0joiRXbPpQ042GgJckkFgw= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/siderolabs/talos/pkg/machinery v1.10.5 h1:R0ZSf9OrHIQ70+LEpQVK/KVCq3E6sUi/OpIjLDVH3Sk= +github.com/siderolabs/talos/pkg/machinery v1.10.5/go.mod h1:GxGnHH6gtX3J9s713+UbKvE9rLnlbYLv+Yn4rqD9Jh0= +github.com/sigstore/protobuf-specs v0.4.1 h1:5SsMqZbdkcO/DNHudaxuCUEjj6x29tS2Xby1BxGU7Zc= +github.com/sigstore/protobuf-specs v0.4.1/go.mod h1:+gXR+38nIa2oEupqDdzg4qSBT0Os+sP7oYv6alWewWc= +github.com/sigstore/rekor v1.3.10 h1:/mSvRo4MZ/59ECIlARhyykAlQlkmeAQpvBPlmJtZOCU= +github.com/sigstore/rekor v1.3.10/go.mod h1:JvryKJ40O0XA48MdzYUPu0y4fyvqt0C4iSY7ri9iu3A= +github.com/sigstore/sigstore v1.9.5 h1:Wm1LT9yF4LhQdEMy5A2JeGRHTrAWGjT3ubE5JUSrGVU= +github.com/sigstore/sigstore v1.9.5/go.mod h1:VtxgvGqCmEZN9X2zhFSOkfXxvKUjpy8RpUW39oCtoII= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= -github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= +github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= -github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA= -github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA= -github.com/thomasten/go-tpm v0.0.0-20230629092004-f43f8e2a59eb h1:840nUyrM9df2aLuzWuIkYx/DrUbX4KQZO6B9LD45aWo= -github.com/thomasten/go-tpm v0.0.0-20230629092004-f43f8e2a59eb/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/theupdateframework/go-tuf v0.7.0 h1:CqbQFrWo1ae3/I0UCblSbczevCCbS31Qvs5LdxRWqRI= +github.com/theupdateframework/go-tuf v0.7.0/go.mod h1:uEB7WSY+7ZIugK6R1hiBMBjQftaFzn7ZCDJcp1tCUug= +github.com/tink-crypto/tink-go/v2 v2.4.0 h1:8VPZeZI4EeZ8P/vB6SIkhlStrJfivTJn+cQ4dtyHNh0= +github.com/tink-crypto/tink-go/v2 v2.4.0/go.mod h1:l//evrF2Y3MjdbpNDNGnKgCpo5zSmvUvnQ4MU+yE2sw= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= -github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810 h1:X6ps8XHfpQjw8dUStzlMi2ybiKQ2Fmdw7UM+TinwvyM= github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810/go.mod h1:dF0BBJ2YrV1+2eAIyEI+KeSidgA6HqoIP1u5XTlMq/o= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -940,229 +794,141 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/zalando/go-keyring v0.2.2 h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4= -github.com/zalando/go-keyring v0.2.2/go.mod h1:sI3evg9Wvpw3+n4SqplGSJUMwtDeROfD4nsFz4z9PG0= -github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA= -github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y= -go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= +github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= +github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= +github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= +go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= +go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= +go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= +go.etcd.io/etcd/client/v3 v3.6.4 h1:YOMrCfMhRzY8NgtzUsHl8hC2EBSnuqbR3dh84Uryl7A= +go.etcd.io/etcd/client/v3 v3.6.4/go.mod h1:jaNNHCyg2FdALyKWnd7hxZXZxZANb0+KGY+YQaEMISo= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 h1:UW0+QyeyBVhn+COBec3nGhfnFe5lwB0ic1JBVjzhk0w= +go.opentelemetry.io/contrib/bridges/prometheus v0.57.0/go.mod h1:ppciCHRLsyCio54qbzQv0E4Jyth/fLWDTJYfvWpcSVk= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/exporters/autoexport v0.57.0 h1:jmTVJ86dP60C01K3slFQa2NQ/Aoi7zA+wy7vMOKD9H4= +go.opentelemetry.io/contrib/exporters/autoexport v0.57.0/go.mod h1:EJBheUMttD/lABFyLXhce47Wr6DPWYReCzaZiXadH7g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 h1:WzNab7hOOLzdDF/EoWCt4glhrbMPVMOO5JYTmpz36Ls= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0/go.mod h1:hKvJwTzJdp90Vh7p6q/9PAOd55dI6WA6sWj62a/JvSs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 h1:S+LdBGiQXtJdowoJoQPEtI52syEP/JYBUpjO49EQhV8= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0/go.mod h1:5KXybFvPGds3QinJWQT7pmXf+TN5YIa7CNYObWRkj50= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 h1:t/Qur3vKSkUCcDVaSumWF2PKHt85pc7fRvFuoVT8qFU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0/go.mod h1:Rl61tySSdcOJWoEgYZVtmnKdA0GeKrSqkHC1t+91CH8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= +go.opentelemetry.io/otel/exporters/prometheus v0.54.0 h1:rFwzp68QMgtzu9PgP3jm9XaMICI6TsofWWPcBDKwlsU= +go.opentelemetry.io/otel/exporters/prometheus v0.54.0/go.mod h1:QyjcV9qDP6VeK5qPyKETvNjmaaEc7+gqjh4SS0ZYzDU= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 h1:CHXNXwfKWfzS65yrlB2PVds1IBZcdsX8Vepy9of0iRU= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0/go.mod h1:zKU4zUgKiaRxrdovSS2amdM5gOc59slmo/zJwGX+YBg= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 h1:cC2yDI3IQd0Udsux7Qmq8ToKAx1XCilTQECZ0KDZyTw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0/go.mod h1:2PD5Ex6z8CFzDbTdOlwyNIUywRr1DN0ospafJM1wJ+s= +go.opentelemetry.io/otel/log v0.8.0 h1:egZ8vV5atrUWUbnSsHn6vB8R21G2wrKqNiDt3iWertk= +go.opentelemetry.io/otel/log v0.8.0/go.mod h1:M9qvDdUTRCopJcGRKg57+JSQ9LgLBrwwfC32epk5NX8= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/log v0.8.0 h1:zg7GUYXqxk1jnGF/dTdLPrK06xJdrXgqgFLnI4Crxvs= +go.opentelemetry.io/otel/sdk/log v0.8.0/go.mod h1:50iXr0UVwQrYS45KbruFrEt4LvAdCaWWgIrsN3ZQggo= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= +go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1170,296 +936,135 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs= -google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/api v0.244.0 h1:lpkP8wVibSKr++NCD36XzTk/IzeKJ3klj7vbj+XU5pE= +google.golang.org/api v0.244.0/go.mod h1:dMVhVcylamkirHdzEBAIQWUCgqY885ivNeZYd7VAVr8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= +google.golang.org/genproto/googleapis/api v0.0.0-20250721164621-a45f3dfb1074 h1:mVXdvnmR3S3BQOqHECm9NGMjYiRtEvDYcqAqedTXY6s= +google.golang.org/genproto/googleapis/api v0.0.0-20250721164621-a45f3dfb1074/go.mod h1:vYFwMYFbmA8vl6Z/krj/h7+U/AqpHknwJX4Uqgfyc7I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 h1:MAKi5q709QWfnkkpNQ0M12hYJ1+e8qYVDyowc4U1XZM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= -gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -helm.sh/helm v2.17.0+incompatible h1:cSe3FaQOpRWLDXvTObQNj0P7WI98IG5yloU6tQVls2k= -helm.sh/helm v2.17.0+incompatible/go.mod h1:0Xbc6ErzwWH9qC55X1+hE3ZwhM3atbhCm/NbFZw5i+4= -helm.sh/helm/v3 v3.13.1 h1:DG+XLGzBJeZvMLlMbm6bPDLV1dGaVW9eZsDoUd1/LM0= -helm.sh/helm/v3 v3.13.1/go.mod h1:TdQRMiq46CSWcc68Hb0uVhvAWusaN90YwAV54cz6JzU= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= -k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/apiserver v0.28.2 h1:rBeYkLvF94Nku9XfXyUIirsVzCzJBs6jMn3NWeHieyI= -k8s.io/apiserver v0.28.2/go.mod h1:f7D5e8wH8MWcKD7azq6Csw9UN+CjdtXIVQUyUhrtb+E= -k8s.io/cli-runtime v0.28.2 h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk= -k8s.io/cli-runtime v0.28.2/go.mod h1:bTpGOvpdsPtDKoyfG4EG041WIyFZLV9qq4rPlkyYfDA= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/cluster-bootstrap v0.27.3 h1:yk1XIWt/mbMgNHFdxd0HyVPq/rnJK7BS3oXj24gHClU= -k8s.io/cluster-bootstrap v0.27.3/go.mod h1:4/bxgDkpV7XPapJS1585P/nvy3FdBIoFssK4Z5oztrc= -k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= -k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM= -k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64= -k8s.io/kubelet v0.27.3 h1:5WhTV1iiBu9q/rr+gvy65LQ+K/e7dmgcaYjys5ipLqY= -k8s.io/kubelet v0.27.3/go.mod h1:Mz42qgZZgWgPmOJEYaR5evmh+EoSwFzEvPBozA2y9mg= -k8s.io/kubernetes v1.27.8 h1:K848lTo/D0jvrxUlTvw4nNADixbhXLHgKNDP/KlFGy8= -k8s.io/kubernetes v1.27.8/go.mod h1:PUXXrx0IhAi+kI9BMDqNJHUnLndVv9W0DkriqyjuJOs= -k8s.io/mount-utils v0.27.3 h1:oubkDKLTZUneW27wgyOmp8a1AAZj04vGmtq+YW8wdvY= -k8s.io/mount-utils v0.27.3/go.mod h1:vmcjYdi2Vg1VTWY7KkhvwJVY6WDHxb/QQhiQKkR8iNs= -k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= -k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= -oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= -sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE= -software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ= +helm.sh/helm/v3 v3.18.4 h1:pNhnHM3nAmDrxz6/UC+hfjDY4yeDATQCka2/87hkZXQ= +helm.sh/helm/v3 v3.18.4/go.mod h1:WVnwKARAw01iEdjpEkP7Ii1tT1pTPYfM1HsakFKM3LI= +k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= +k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= +k8s.io/apiextensions-apiserver v0.33.3 h1:qmOcAHN6DjfD0v9kxL5udB27SRP6SG/MTopmge3MwEs= +k8s.io/apiextensions-apiserver v0.33.3/go.mod h1:oROuctgo27mUsyp9+Obahos6CWcMISSAPzQ77CAQGz8= +k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= +k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.3 h1:Wv0hGc+QFdMJB4ZSiHrCgN3zL3QRatu56+rpccKC3J4= +k8s.io/apiserver v0.33.3/go.mod h1:05632ifFEe6TxwjdAIrwINHWE2hLwyADFk5mBsQa15E= +k8s.io/cli-runtime v0.33.2 h1:koNYQKSDdq5AExa/RDudXMhhtFasEg48KLS2KSAU74Y= +k8s.io/cli-runtime v0.33.2/go.mod h1:gnhsAWpovqf1Zj5YRRBBU7PFsRc6NkEkwYNQE+mXL88= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= +k8s.io/cluster-bootstrap v0.33.3 h1:u2NTxJ5CFSBFXaDxLQoOWMly8eni31psVso+caq6uwI= +k8s.io/cluster-bootstrap v0.33.3/go.mod h1:p970f8u8jf273zyQ5raD8WUu2XyAl0SAWOY82o7i/ds= +k8s.io/component-base v0.33.3 h1:mlAuyJqyPlKZM7FyaoM/LcunZaaY353RXiOd2+B5tGA= +k8s.io/component-base v0.33.3/go.mod h1:ktBVsBzkI3imDuxYXmVxZ2zxJnYTZ4HAsVj9iF09qp4= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/kubectl v0.33.2 h1:7XKZ6DYCklu5MZQzJe+CkCjoGZwD1wWl7t/FxzhMz7Y= +k8s.io/kubectl v0.33.2/go.mod h1:8rC67FB8tVTYraovAGNi/idWIK90z2CHFNMmGJZJ3KI= +k8s.io/kubelet v0.33.3 h1:Cvy8+7Lq9saZds2ib7YBXbKvkMMJu3f5mzucmhSIJno= +k8s.io/kubelet v0.33.3/go.mod h1:Q1Cfr6VQq1m9v9XsE/mDmhTxPdN6NPU6Ug0e6mAqi58= +k8s.io/kubernetes v1.33.3 h1:dBx5Z2ZhR8kNzAwCoCz4j1niUbUrNUDVxeSj4/Ienu0= +k8s.io/kubernetes v1.33.3/go.mod h1:nrt8sldmckKz2fCZhgRX3SKfS2e+CzXATPv6ITNkU00= +k8s.io/mount-utils v0.33.3 h1:Q1jsnqdS4LdtJSYSXgiQv/XNrRHQncLk3gMYjKNSZrE= +k8s.io/mount-utils v0.33.3/go.mod h1:1JR4rKymg8B8bCPo618hpSAdrpO6XLh0Acqok/xVwPE= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +libvirt.org/go/libvirt v1.11005.0 h1:3ff08ii8h9XSe0OPmTCSUJ/3Nj+ssCAYhC6MK6ppwow= +libvirt.org/go/libvirt v1.11005.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ= +oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= +oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ= +sigs.k8s.io/kustomize/api v0.19.0/go.mod h1:/BbwnivGVcBh1r+8m3tH1VNxJmHSk1PzP5fkP6lbL1o= +sigs.k8s.io/kustomize/kyaml v0.19.0 h1:RFge5qsO1uHhwJsu3ipV7RNolC7Uozc0jUBC/61XSlA= +sigs.k8s.io/kustomize/kyaml v0.19.0/go.mod h1:FeKD5jEOH+FbZPpqUghBP8mrLjJ3+zD3/rf9NNu1cwY= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/go.work b/go.work index cf4aac99b..0d9909f84 100644 --- a/go.work +++ b/go.work @@ -1,13 +1,8 @@ -go 1.21 +go 1.24.4 -toolchain go1.21.5 +toolchain go1.24.4 use ( . - ./3rdparty/node-maintenance-operator - ./hack ./hack/tools - ./operators/constellation-node-operator - ./operators/constellation-node-operator/api - ./terraform-provider-constellation ) diff --git a/hack/bazel-deps-mirror/BUILD.bazel b/hack/bazel-deps-mirror/BUILD.bazel index 0ccbc3e61..c4880cc83 100644 --- a/hack/bazel-deps-mirror/BUILD.bazel +++ b/hack/bazel-deps-mirror/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//internal/logger", "@com_github_bazelbuild_buildtools//build", "@com_github_spf13_cobra//:cobra", - "@org_uber_go_zap//zapcore", ], ) diff --git a/hack/bazel-deps-mirror/README.md b/hack/bazel-deps-mirror/README.md index 9aa5fb21e..354e506ab 100644 --- a/hack/bazel-deps-mirror/README.md +++ b/hack/bazel-deps-mirror/README.md @@ -2,7 +2,7 @@ This directory contains tooling to automatically mirror the dependencies of a Bazel project into the Constellation CDN at `https://cdn.confidential.cloud/`. -The tool searches for various rules in the WORKSPACE.bazel file and all loaded .bzl files. +The tool searches for various rules in the WORKSPACE.bzlmod file and all loaded .bzl files. It has the following commands: - check: checks if the dependencies all have a mirror URL and optionally checks if the mirror really returns the expected file diff --git a/hack/bazel-deps-mirror/bazel-deps-mirror.go b/hack/bazel-deps-mirror/bazel-deps-mirror.go index 5e2d92617..b99f75e39 100644 --- a/hack/bazel-deps-mirror/bazel-deps-mirror.go +++ b/hack/bazel-deps-mirror/bazel-deps-mirror.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // bazel-deps-mirror adds external dependencies to edgeless systems' mirror. diff --git a/hack/bazel-deps-mirror/check.go b/hack/bazel-deps-mirror/check.go index c8089ff73..30c65a131 100644 --- a/hack/bazel-deps-mirror/check.go +++ b/hack/bazel-deps-mirror/check.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -9,6 +9,8 @@ package main import ( "context" "errors" + "fmt" + "log/slog" "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/bazelfiles" "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/issues" @@ -16,7 +18,6 @@ import ( "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/rules" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newCheckCmd() *cobra.Command { @@ -38,15 +39,15 @@ func runCheck(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "mirror", flags.mirror, "mirrorUnauthenticated", flags.mirrorUnauthenticated) filesHelper, err := bazelfiles.New() if err != nil { return err } - log.Debugf("Searching for Bazel files in the current WORKSPACE and all subdirectories...") + log.Debug("Searching for Bazel files in the current WORKSPACE and all subdirectories...") bazelFiles, err := filesHelper.FindFiles() if err != nil { return err @@ -55,10 +56,10 @@ func runCheck(cmd *cobra.Command, _ []string) error { var mirrorCheck mirrorChecker switch { case flags.mirrorUnauthenticated: - log.Debugf("Checking consistency of all referenced CAS objects without authentication.") + log.Debug("Checking consistency of all referenced CAS objects without authentication.") mirrorCheck = mirror.NewUnauthenticated(flags.mirrorBaseURL, mirror.Run, log) case flags.mirror: - log.Debugf("Checking consistency of all referenced CAS objects using AWS S3.") + log.Debug("Checking consistency of all referenced CAS objects using AWS S3.") mirrorCheck, err = mirror.New(cmd.Context(), flags.region, flags.bucket, flags.mirrorBaseURL, mirror.Run, log) if err != nil { return err @@ -78,17 +79,17 @@ func runCheck(cmd *cobra.Command, _ []string) error { } } if len(iss) > 0 { - log.Infof("Found issues in rules") + log.Info("Found issues in rules") iss.Report(cmd.OutOrStdout()) return errors.New("found issues in rules") } - log.Infof("No issues found 🦭") + log.Info("No issues found 🦭") return nil } -func checkBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorCheck mirrorChecker, bazelFile bazelfiles.BazelFile, log *logger.Logger) (issByFile issues.ByFile, err error) { - log.Debugf("Checking file: %s", bazelFile.RelPath) +func checkBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorCheck mirrorChecker, bazelFile bazelfiles.BazelFile, log *slog.Logger) (issByFile issues.ByFile, err error) { + log.Debug(fmt.Sprintf("Checking file: %q", bazelFile.RelPath)) issByFile = issues.NewByFile() buildfile, err := fileHelper.LoadFile(bazelFile) if err != nil { @@ -96,12 +97,12 @@ func checkBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorCh } found := rules.Rules(buildfile, rules.SupportedRules) if len(found) == 0 { - log.Debugf("No rules found in file: %s", bazelFile.RelPath) + log.Debug(fmt.Sprintf("No rules found in file: %q", bazelFile.RelPath)) return issByFile, nil } - log.Debugf("Found %d rules in file: %s", len(found), bazelFile.RelPath) + log.Debug(fmt.Sprintf("Found %d rules in file: %q", len(found), bazelFile.RelPath)) for _, rule := range found { - log.Debugf("Checking rule: %s", rule.Name()) + log.Debug(fmt.Sprintf("Checking rule: %q", rule.Name())) // check if the rule is a valid pinned dependency rule (has all required attributes) if issues := rules.ValidatePinned(rule); len(issues) > 0 { issByFile.Add(rule.Name(), issues...) @@ -130,7 +131,7 @@ type checkFlags struct { region string bucket string mirrorBaseURL string - logLevel zapcore.Level + logLevel slog.Level } func parseCheckFlags(cmd *cobra.Command) (checkFlags, error) { @@ -146,9 +147,9 @@ func parseCheckFlags(cmd *cobra.Command) (checkFlags, error) { if err != nil { return checkFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } region, err := cmd.Flags().GetString("region") if err != nil { diff --git a/hack/bazel-deps-mirror/fix.go b/hack/bazel-deps-mirror/fix.go index c40d42d05..a6018dfe1 100644 --- a/hack/bazel-deps-mirror/fix.go +++ b/hack/bazel-deps-mirror/fix.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -9,6 +9,8 @@ package main import ( "context" "errors" + "fmt" + "log/slog" "github.com/bazelbuild/buildtools/build" "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/bazelfiles" @@ -17,7 +19,6 @@ import ( "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/rules" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newFixCmd() *cobra.Command { @@ -38,15 +39,15 @@ func runFix(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "unauthenticated", flags.unauthenticated, "dryRun", flags.dryRun) fileHelper, err := bazelfiles.New() if err != nil { return err } - log.Debugf("Searching for Bazel files in the current WORKSPACE and all subdirectories...") + log.Debug("Searching for Bazel files in the current WORKSPACE and all subdirectories...") bazelFiles, err := fileHelper.FindFiles() if err != nil { return err @@ -55,10 +56,10 @@ func runFix(cmd *cobra.Command, _ []string) error { var mirrorUpload mirrorUploader switch { case flags.unauthenticated: - log.Warnf("Fixing rules without authentication for AWS S3. If artifacts are not yet mirrored, this will fail.") + log.Warn("Fixing rules without authentication for AWS S3. If artifacts are not yet mirrored, this will fail.") mirrorUpload = mirror.NewUnauthenticated(flags.mirrorBaseURL, flags.dryRun, log) default: - log.Debugf("Fixing rules with authentication for AWS S3.") + log.Debug("Fixing rules with authentication for AWS S3.") mirrorUpload, err = mirror.New(cmd.Context(), flags.region, flags.bucket, flags.mirrorBaseURL, flags.dryRun, log) if err != nil { return err @@ -76,29 +77,29 @@ func runFix(cmd *cobra.Command, _ []string) error { } } if len(issues) > 0 { - log.Warnf("Found %d unfixable issues in rules", len(issues)) + log.Warn(fmt.Sprintf("Found %d unfixable issues in rules", len(issues))) issues.Report(cmd.OutOrStdout()) return errors.New("found issues in rules") } - log.Infof("No unfixable issues found") + log.Info("No unfixable issues found") return nil } -func fixBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUpload mirrorUploader, bazelFile bazelfiles.BazelFile, dryRun bool, log *logger.Logger) (iss issues.ByFile, err error) { +func fixBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUpload mirrorUploader, bazelFile bazelfiles.BazelFile, dryRun bool, log *slog.Logger) (iss issues.ByFile, err error) { iss = issues.NewByFile() var changed bool // true if any rule in this file was changed - log.Infof("Checking file: %s", bazelFile.RelPath) + log.Info(fmt.Sprintf("Checking file: %s", bazelFile.RelPath)) buildfile, err := fileHelper.LoadFile(bazelFile) if err != nil { return iss, err } found := rules.Rules(buildfile, rules.SupportedRules) if len(found) == 0 { - log.Debugf("No rules found in file: %s", bazelFile.RelPath) + log.Debug(fmt.Sprintf("No rules found in file: %q", bazelFile.RelPath)) return iss, nil } - log.Debugf("Found %d rules in file: %s", len(found), bazelFile.RelPath) + log.Debug(fmt.Sprintf("Found %d rules in file: %q", len(found), bazelFile.RelPath)) for _, rule := range found { changedRule, ruleIssues := fixRule(ctx, mirrorUpload, rule, log) if len(ruleIssues) > 0 { @@ -108,11 +109,11 @@ func fixBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUplo } if len(iss) > 0 { - log.Warnf("File %s has issues. Not saving!", bazelFile.RelPath) + log.Warn(fmt.Sprintf("File %s has issues. Not saving!", bazelFile.RelPath)) return iss, nil } if !changed { - log.Debugf("No changes to file: %s", bazelFile.RelPath) + log.Debug(fmt.Sprintf("No changes to file: %q", bazelFile.RelPath)) return iss, nil } if dryRun { @@ -120,10 +121,10 @@ func fixBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUplo if err != nil { return iss, err } - log.Infof("Dry run: would save updated file %s with diff:\n%s", bazelFile.RelPath, diff) + log.Info(fmt.Sprintf("Dry run: would save updated file %s with diff:\n%s", bazelFile.RelPath, diff)) return iss, nil } - log.Infof("Saving updated file: %s", bazelFile.RelPath) + log.Info(fmt.Sprintf("Saving updated file: %s", bazelFile.RelPath)) if err := fileHelper.WriteFile(bazelFile, buildfile); err != nil { return iss, err } @@ -131,7 +132,7 @@ func fixBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUplo return iss, nil } -func learnHashForRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *logger.Logger) error { +func learnHashForRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *slog.Logger) error { upstreamURLs, err := rules.UpstreamURLs(rule) if err != nil { return err @@ -141,12 +142,12 @@ func learnHashForRule(ctx context.Context, mirrorUpload mirrorUploader, rule *bu return err } rules.SetHash(rule, learnedHash) - log.Debugf("Learned hash for rule %s: %s", rule.Name(), learnedHash) + log.Debug(fmt.Sprintf("Learned hash for rule %q: %q", rule.Name(), learnedHash)) return nil } -func fixRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *logger.Logger) (changed bool, iss []error) { - log.Debugf("Fixing rule: %s", rule.Name()) +func fixRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *slog.Logger) (changed bool, iss []error) { + log.Debug(fmt.Sprintf("Fixing rule: %q", rule.Name())) // try to learn the hash if hash, err := rules.GetHash(rule); err != nil || hash == "" { @@ -182,14 +183,14 @@ func fixRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, } if checkErr := mirrorUpload.Check(ctx, expectedHash); checkErr != nil { - log.Infof("Artifact %s with hash %s is not yet mirrored. Uploading...", rule.Name(), expectedHash) + log.Info(fmt.Sprintf("Artifact %s with hash %s is not yet mirrored. Uploading...", rule.Name(), expectedHash)) if uploadErr := mirrorUpload.Mirror(ctx, expectedHash, rules.GetURLs(rule)); uploadErr != nil { // don't try to fix the rule if the upload failed iss = append(iss, uploadErr) return changed, iss } } else { - log.Infof("Artifact %s with hash %s was already uploaded before. Adding to rule...", rule.Name(), expectedHash) + log.Info(fmt.Sprintf("Artifact %s with hash %s was already uploaded before. Adding to rule...", rule.Name(), expectedHash)) } // now the artifact is mirrored (if it wasn't already) and we can fix the rule @@ -211,7 +212,7 @@ type fixFlags struct { region string bucket string mirrorBaseURL string - logLevel zapcore.Level + logLevel slog.Level } func parseFixFlags(cmd *cobra.Command) (fixFlags, error) { @@ -227,9 +228,9 @@ func parseFixFlags(cmd *cobra.Command) (fixFlags, error) { if err != nil { return fixFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } region, err := cmd.Flags().GetString("region") if err != nil { diff --git a/hack/bazel-deps-mirror/internal/bazelfiles/files.go b/hack/bazel-deps-mirror/internal/bazelfiles/files.go index 468ff24ac..a891deb49 100644 --- a/hack/bazel-deps-mirror/internal/bazelfiles/files.go +++ b/hack/bazel-deps-mirror/internal/bazelfiles/files.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // package bazelfiles is used to find and handle Bazel WORKSPACE and bzl files. @@ -53,12 +53,12 @@ func (h *Helper) FindFiles() ([]BazelFile, error) { return append(bzlFiles, workspaceFile), nil } -// findWorkspaceFile returns the path to the Bazel WORKSPACE.bazel file (or WORKSPACE if the former doesn't exist). +// findWorkspaceFile returns the path to the Bazel WORKSPACE.bzlmod file (or WORKSPACE if the former doesn't exist). func (h *Helper) findWorkspaceFile() (BazelFile, error) { - if _, err := h.fs.Stat("WORKSPACE.bazel"); err == nil { + if _, err := h.fs.Stat("WORKSPACE.bzlmod"); err == nil { return BazelFile{ - RelPath: "WORKSPACE.bazel", - AbsPath: filepath.Join(h.workspaceRoot, "WORKSPACE.bazel"), + RelPath: "WORKSPACE.bzlmod", + AbsPath: filepath.Join(h.workspaceRoot, "WORKSPACE.bzlmod"), Type: BazelFileTypeWorkspace, }, nil } @@ -150,8 +150,10 @@ type BazelFile struct { type BazelFileType int const ( - BazelFileTypeBzl = iota // BazelFileTypeBzl is a .bzl file - BazelFileTypeWorkspace // BazelFileTypeWorkspace is a WORKSPACE or WORKSPACE.bazel file + // BazelFileTypeBzl is a .bzl file. + BazelFileTypeBzl = iota + // BazelFileTypeWorkspace is a WORKSPACE or WORKSPACE.bzlmod file. + BazelFileTypeWorkspace ) // LookupEnv can be the real os.LookupEnv or a mock for testing. diff --git a/hack/bazel-deps-mirror/internal/bazelfiles/files_test.go b/hack/bazel-deps-mirror/internal/bazelfiles/files_test.go index 61add0b32..889a490ec 100644 --- a/hack/bazel-deps-mirror/internal/bazelfiles/files_test.go +++ b/hack/bazel-deps-mirror/internal/bazelfiles/files_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package bazelfiles @@ -18,7 +18,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestFindFiles(t *testing.T) { @@ -42,22 +42,22 @@ func TestFindFiles(t *testing.T) { }, }, }, - "only WORKSPACE.bazel file": { - files: []string{"WORKSPACE.bazel"}, + "only WORKSPACE.bzlmod file": { + files: []string{"WORKSPACE.bzlmod"}, wantFiles: []BazelFile{ { - RelPath: "WORKSPACE.bazel", - AbsPath: "/WORKSPACE.bazel", + RelPath: "WORKSPACE.bzlmod", + AbsPath: "/WORKSPACE.bzlmod", Type: BazelFileTypeWorkspace, }, }, }, - "both WORKSPACE and WORKSPACE.bazel files": { - files: []string{"WORKSPACE", "WORKSPACE.bazel"}, + "both WORKSPACE and WORKSPACE.bzlmod files": { + files: []string{"WORKSPACE", "WORKSPACE.bzlmod"}, wantFiles: []BazelFile{ { - RelPath: "WORKSPACE.bazel", - AbsPath: "/WORKSPACE.bazel", + RelPath: "WORKSPACE.bzlmod", + AbsPath: "/WORKSPACE.bzlmod", Type: BazelFileTypeWorkspace, }, }, @@ -67,11 +67,11 @@ func TestFindFiles(t *testing.T) { wantErr: true, }, "all kinds": { - files: []string{"WORKSPACE", "WORKSPACE.bazel", "foo.bzl", "bar.bzl", "unused.txt", "folder/baz.bzl"}, + files: []string{"WORKSPACE", "WORKSPACE.bzlmod", "foo.bzl", "bar.bzl", "unused.txt", "folder/baz.bzl"}, wantFiles: []BazelFile{ { - RelPath: "WORKSPACE.bazel", - AbsPath: "/WORKSPACE.bazel", + RelPath: "WORKSPACE.bzlmod", + AbsPath: "/WORKSPACE.bzlmod", Type: BazelFileTypeWorkspace, }, { @@ -216,15 +216,15 @@ func TestDiff(t *testing.T) { assert := assert.New(t) require := require.New(t) fs := afero.NewMemMapFs() - err := afero.WriteFile(fs, "WORKSPACE.bazel", []byte(""), 0o644) + err := afero.WriteFile(fs, "WORKSPACE.bzlmod", []byte(""), 0o644) require.NoError(err) helper := Helper{ fs: fs, workspaceRoot: "/", } fileRef := BazelFile{ - RelPath: "WORKSPACE.bazel", - AbsPath: "/WORKSPACE.bazel", + RelPath: "WORKSPACE.bzlmod", + AbsPath: "/WORKSPACE.bzlmod", Type: BazelFileTypeWorkspace, } bf, err := helper.LoadFile(fileRef) @@ -247,10 +247,10 @@ func TestDiff(t *testing.T) { ) diff, err = helper.Diff(fileRef, bf) require.NoError(err) - assert.Equal("--- a/WORKSPACE.bazel\n+++ b/WORKSPACE.bazel\n@@ -1 +1 @@\n+workspace(name = \"foo\")\n", diff) + assert.Equal("--- a/WORKSPACE.bzlmod\n+++ b/WORKSPACE.bzlmod\n@@ -1 +1 @@\n+workspace(name = \"foo\")\n", diff) err = helper.WriteFile(fileRef, bf) require.NoError(err) - contents, err := afero.ReadFile(fs, "WORKSPACE.bazel") + contents, err := afero.ReadFile(fs, "WORKSPACE.bzlmod") assert.NoError(err) assert.Equal("workspace(name = \"foo\")\n", string(contents)) diff, err = helper.Diff(fileRef, bf) diff --git a/hack/bazel-deps-mirror/internal/issues/issues.go b/hack/bazel-deps-mirror/internal/issues/issues.go index f01495ceb..a1cac0e64 100644 --- a/hack/bazel-deps-mirror/internal/issues/issues.go +++ b/hack/bazel-deps-mirror/internal/issues/issues.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // package issues can store and report issues found during the bazel-deps-mirror process. diff --git a/hack/bazel-deps-mirror/internal/issues/issues_test.go b/hack/bazel-deps-mirror/internal/issues/issues_test.go index 614416dda..6e402f1bb 100644 --- a/hack/bazel-deps-mirror/internal/issues/issues_test.go +++ b/hack/bazel-deps-mirror/internal/issues/issues_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package issues @@ -16,7 +16,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestMap(t *testing.T) { diff --git a/hack/bazel-deps-mirror/internal/mirror/BUILD.bazel b/hack/bazel-deps-mirror/internal/mirror/BUILD.bazel index 88e4e4c9e..5e4b25479 100644 --- a/hack/bazel-deps-mirror/internal/mirror/BUILD.bazel +++ b/hack/bazel-deps-mirror/internal/mirror/BUILD.bazel @@ -7,7 +7,6 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/mirror", visibility = ["//hack/bazel-deps-mirror:__subpackages__"], deps = [ - "//internal/logger", "@com_github_aws_aws_sdk_go_v2_config//:config", "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager", "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", diff --git a/hack/bazel-deps-mirror/internal/mirror/mirror.go b/hack/bazel-deps-mirror/internal/mirror/mirror.go index 9c1415242..947f565da 100644 --- a/hack/bazel-deps-mirror/internal/mirror/mirror.go +++ b/hack/bazel-deps-mirror/internal/mirror/mirror.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // package mirror is used upload and download Bazel dependencies to and from a mirror. @@ -15,6 +15,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "net/url" "path" @@ -23,7 +24,6 @@ import ( s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/edgelesssys/constellation/v2/internal/logger" ) // Maintainer can upload and download files to and from a CAS mirror. @@ -39,11 +39,11 @@ type Maintainer struct { unauthenticated bool dryRun bool - log *logger.Logger + log *slog.Logger } // NewUnauthenticated creates a new Maintainer that dose not require authentication can only download files from a CAS mirror. -func NewUnauthenticated(mirrorBaseURL string, dryRun bool, log *logger.Logger) *Maintainer { +func NewUnauthenticated(mirrorBaseURL string, dryRun bool, log *slog.Logger) *Maintainer { return &Maintainer{ httpClient: http.DefaultClient, mirrorBaseURL: mirrorBaseURL, @@ -54,7 +54,7 @@ func NewUnauthenticated(mirrorBaseURL string, dryRun bool, log *logger.Logger) * } // New creates a new Maintainer that can upload and download files to and from a CAS mirror. -func New(ctx context.Context, region, bucket, mirrorBaseURL string, dryRun bool, log *logger.Logger) (*Maintainer, error) { +func New(ctx context.Context, region, bucket, mirrorBaseURL string, dryRun bool, log *slog.Logger) (*Maintainer, error) { cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) if err != nil { return nil, err @@ -95,17 +95,17 @@ func (m *Maintainer) Mirror(ctx context.Context, hash string, urls []string) err } for _, url := range urls { - m.log.Debugf("Mirroring file with hash %v from %q", hash, url) + m.log.Debug(fmt.Sprintf("Mirroring file with hash %q from %q", hash, url)) body, err := m.downloadFromUpstream(ctx, url) if err != nil { - m.log.Debugf("Failed to download file from %q: %v", url, err) + m.log.Debug(fmt.Sprintf("Failed to download file from %q: %q", url, err)) continue } defer body.Close() streamedHash := sha256.New() tee := io.TeeReader(body, streamedHash) if err := m.put(ctx, hash, tee); err != nil { - m.log.Warnf("Failed to stream file from upstream %q to mirror: %v.. Trying next url.", url, err) + m.log.Warn(fmt.Sprintf("Failed to stream file from upstream %q to mirror: %v.. Trying next url.", url, err)) continue } actualHash := hex.EncodeToString(streamedHash.Sum(nil)) @@ -117,7 +117,7 @@ func (m *Maintainer) Mirror(ctx context.Context, hash string, urls []string) err if err != nil { return err } - m.log.Debugf("File uploaded successfully to mirror from %q as %q", url, pubURL) + m.log.Debug(fmt.Sprintf("File uploaded successfully to mirror from %q as %q", url, pubURL)) return nil } return fmt.Errorf("failed to download / reupload file with hash %v from any of the urls: %v", hash, urls) @@ -126,19 +126,19 @@ func (m *Maintainer) Mirror(ctx context.Context, hash string, urls []string) err // Learn downloads a file from one of the existing (non-mirror) urls, hashes it and returns the hash. func (m *Maintainer) Learn(ctx context.Context, urls []string) (string, error) { for _, url := range urls { - m.log.Debugf("Learning new hash from %q", url) + m.log.Debug(fmt.Sprintf("Learning new hash from %q", url)) body, err := m.downloadFromUpstream(ctx, url) if err != nil { - m.log.Debugf("Failed to download file from %q: %v", url, err) + m.log.Debug(fmt.Sprintf("Failed to download file from %q: %q", url, err)) continue } defer body.Close() streamedHash := sha256.New() if _, err := io.Copy(streamedHash, body); err != nil { - m.log.Debugf("Failed to stream file from %q: %v", url, err) + m.log.Debug(fmt.Sprintf("Failed to stream file from %q: %q", url, err)) } learnedHash := hex.EncodeToString(streamedHash.Sum(nil)) - m.log.Debugf("File successfully downloaded from %q with %q", url, learnedHash) + m.log.Debug(fmt.Sprintf("File successfully downloaded from %q with %q", url, learnedHash)) return learnedHash, nil } return "", fmt.Errorf("failed to download file / learn hash from any of the urls: %v", urls) @@ -146,7 +146,7 @@ func (m *Maintainer) Learn(ctx context.Context, urls []string) (string, error) { // Check checks if a file is present and has the correct hash in the CAS mirror. func (m *Maintainer) Check(ctx context.Context, expectedHash string) error { - m.log.Debugf("Checking consistency of object with hash %v", expectedHash) + m.log.Debug(fmt.Sprintf("Checking consistency of object with hash %q", expectedHash)) if m.unauthenticated { return m.checkUnauthenticated(ctx, expectedHash) } @@ -157,7 +157,7 @@ func (m *Maintainer) Check(ctx context.Context, expectedHash string) error { // It uses the authenticated CAS s3 endpoint to download the file metadata. func (m *Maintainer) checkAuthenticated(ctx context.Context, expectedHash string) error { key := path.Join(keyBase, expectedHash) - m.log.Debugf("Check: s3 getObjectAttributes {Bucket: %v, Key: %v}", m.bucket, key) + m.log.Debug(fmt.Sprintf("Check: s3 getObjectAttributes {Bucket: %q, Key: %q}", m.bucket, key)) attributes, err := m.objectStorageClient.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ Bucket: &m.bucket, Key: &key, @@ -168,13 +168,13 @@ func (m *Maintainer) checkAuthenticated(ctx context.Context, expectedHash string } hasChecksum := attributes.Checksum != nil && attributes.Checksum.ChecksumSHA256 != nil && len(*attributes.Checksum.ChecksumSHA256) > 0 - isSinglePart := attributes.ObjectParts == nil || attributes.ObjectParts.TotalPartsCount == 1 + isSinglePart := attributes.ObjectParts == nil || attributes.ObjectParts.TotalPartsCount == nil || *attributes.ObjectParts.TotalPartsCount == 1 if !hasChecksum || !isSinglePart { // checksums are not guaranteed to be present // and if present, they are only meaningful for single part objects // fallback if checksum cannot be verified from attributes - m.log.Debugf("S3 object attributes cannot be used to verify key %v. Falling back to download.", key) + m.log.Debug(fmt.Sprintf("S3 object attributes cannot be used to verify key %q. Falling back to download.", key)) return m.checkUnauthenticated(ctx, expectedHash) } @@ -192,7 +192,7 @@ func (m *Maintainer) checkUnauthenticated(ctx context.Context, expectedHash stri if err != nil { return err } - m.log.Debugf("Check: http get {Url: %v}", pubURL) + m.log.Debug(fmt.Sprintf("Check: http get {Url: %q}", pubURL)) req, err := http.NewRequestWithContext(ctx, http.MethodGet, pubURL, http.NoBody) if err != nil { return err @@ -221,10 +221,10 @@ func (m *Maintainer) put(ctx context.Context, hash string, data io.Reader) error key := path.Join(keyBase, hash) if m.dryRun { - m.log.Debugf("DryRun: s3 put object {Bucket: %v, Key: %v}", m.bucket, key) + m.log.Debug(fmt.Sprintf("DryRun: s3 put object {Bucket: %q, Key: %q}", m.bucket, key)) return nil } - m.log.Debugf("Uploading object with hash %v to s3://%v/%v", hash, m.bucket, key) + m.log.Debug(fmt.Sprintf("Uploading object with hash %q to \"s3://%s/%s\"", hash, m.bucket, key)) _, err := m.uploadClient.Upload(ctx, &s3.PutObjectInput{ Bucket: &m.bucket, Key: &key, diff --git a/hack/bazel-deps-mirror/internal/mirror/mirror_test.go b/hack/bazel-deps-mirror/internal/mirror/mirror_test.go index c4f52ff08..c94a84507 100644 --- a/hack/bazel-deps-mirror/internal/mirror/mirror_test.go +++ b/hack/bazel-deps-mirror/internal/mirror/mirror_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package mirror @@ -24,7 +24,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestMirrorURL(t *testing.T) { @@ -137,7 +137,7 @@ func TestMirror(t *testing.T) { unauthenticated: tc.unauthenticated, log: logger.NewTest(t), } - err := m.Mirror(context.Background(), tc.hash, []string{tc.upstreamURL}) + err := m.Mirror(t.Context(), tc.hash, []string{tc.upstreamURL}) if tc.wantErr { assert.Error(t, err) } else { @@ -180,7 +180,7 @@ func TestLearn(t *testing.T) { }, log: logger.NewTest(t), } - gotHash, err := m.Learn(context.Background(), []string{"https://example.com/foo"}) + gotHash, err := m.Learn(t.Context(), []string{"https://example.com/foo"}) if tc.wantErr { assert.Error(err) return @@ -230,7 +230,7 @@ func TestCheck(t *testing.T) { ChecksumSHA256: toPtr("tcH7Lvxta0Z0wv3MSM4BtDo7fAN2PAwzVd4Ame4PjHM="), }, ObjectParts: &types.GetObjectAttributesParts{ - TotalPartsCount: 1, + TotalPartsCount: toPtr(int32(1)), }, }, wantErr: true, @@ -242,7 +242,7 @@ func TestCheck(t *testing.T) { ChecksumSHA256: toPtr("LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564="), }, ObjectParts: &types.GetObjectAttributesParts{ - TotalPartsCount: 1, + TotalPartsCount: toPtr(int32(1)), }, }, }, @@ -250,7 +250,7 @@ func TestCheck(t *testing.T) { hash: "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", authenticatedResponse: &s3.GetObjectAttributesOutput{ ObjectParts: &types.GetObjectAttributesParts{ - TotalPartsCount: 2, + TotalPartsCount: toPtr(int32(2)), }, }, unauthenticatedResponse: []byte("foo"), @@ -274,7 +274,7 @@ func TestCheck(t *testing.T) { }, log: logger.NewTest(t), } - err := m.Check(context.Background(), tc.hash) + err := m.Check(t.Context(), tc.hash) if tc.wantErr { assert.Error(t, err) } else { diff --git a/hack/bazel-deps-mirror/internal/rules/rules.go b/hack/bazel-deps-mirror/internal/rules/rules.go index d8a5269f3..5cb90ccaa 100644 --- a/hack/bazel-deps-mirror/internal/rules/rules.go +++ b/hack/bazel-deps-mirror/internal/rules/rules.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // package rules is used find and modify Bazel rules in WORKSPACE and bzl files. diff --git a/hack/bazel-deps-mirror/internal/rules/rules_test.go b/hack/bazel-deps-mirror/internal/rules/rules_test.go index 60c56bf0f..4e494d0a3 100644 --- a/hack/bazel-deps-mirror/internal/rules/rules_test.go +++ b/hack/bazel-deps-mirror/internal/rules/rules_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package rules @@ -16,7 +16,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestRules(t *testing.T) { diff --git a/hack/bazel-deps-mirror/upgrade.go b/hack/bazel-deps-mirror/upgrade.go index 8e2af75f0..e4ceca996 100644 --- a/hack/bazel-deps-mirror/upgrade.go +++ b/hack/bazel-deps-mirror/upgrade.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -9,6 +9,8 @@ package main import ( "context" "errors" + "fmt" + "log/slog" "github.com/bazelbuild/buildtools/build" "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/bazelfiles" @@ -17,7 +19,6 @@ import ( "github.com/edgelesssys/constellation/v2/hack/bazel-deps-mirror/internal/rules" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newUpgradeCmd() *cobra.Command { @@ -38,15 +39,15 @@ func runUpgrade(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "unauthenticated", flags.unauthenticated, "dryRun", flags.dryRun) fileHelper, err := bazelfiles.New() if err != nil { return err } - log.Debugf("Searching for Bazel files in the current WORKSPACE and all subdirectories...") + log.Debug("Searching for Bazel files in the current WORKSPACE and all subdirectories...") bazelFiles, err := fileHelper.FindFiles() if err != nil { return err @@ -55,10 +56,10 @@ func runUpgrade(cmd *cobra.Command, _ []string) error { var mirrorUpload mirrorUploader switch { case flags.unauthenticated: - log.Warnf("Upgrading rules without authentication for AWS S3. If artifacts are not yet mirrored, this will fail.") + log.Warn("Upgrading rules without authentication for AWS S3. If artifacts are not yet mirrored, this will fail.") mirrorUpload = mirror.NewUnauthenticated(flags.mirrorBaseURL, flags.dryRun, log) default: - log.Debugf("Upgrading rules with authentication for AWS S3.") + log.Debug("Upgrading rules with authentication for AWS S3.") mirrorUpload, err = mirror.New(cmd.Context(), flags.region, flags.bucket, flags.mirrorBaseURL, flags.dryRun, log) if err != nil { return err @@ -76,29 +77,29 @@ func runUpgrade(cmd *cobra.Command, _ []string) error { } } if len(issues) > 0 { - log.Warnf("Found %d issues in rules", len(issues)) + log.Warn(fmt.Sprintf("Found %d issues in rules", len(issues))) issues.Report(cmd.OutOrStdout()) return errors.New("found issues in rules") } - log.Infof("No issues found") + log.Info("No issues found") return nil } -func upgradeBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUpload mirrorUploader, bazelFile bazelfiles.BazelFile, dryRun bool, log *logger.Logger) (iss issues.ByFile, err error) { +func upgradeBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirrorUpload mirrorUploader, bazelFile bazelfiles.BazelFile, dryRun bool, log *slog.Logger) (iss issues.ByFile, err error) { iss = issues.NewByFile() var changed bool // true if any rule in this file was changed - log.Infof("Checking file: %s", bazelFile.RelPath) + log.Info(fmt.Sprintf("Checking file: %s", bazelFile.RelPath)) buildfile, err := fileHelper.LoadFile(bazelFile) if err != nil { return iss, err } found := rules.Rules(buildfile, rules.SupportedRules) if len(found) == 0 { - log.Debugf("No rules found in file: %s", bazelFile.RelPath) + log.Debug(fmt.Sprintf("No rules found in file: %q", bazelFile.RelPath)) return iss, nil } - log.Debugf("Found %d rules in file: %s", len(found), bazelFile.RelPath) + log.Debug(fmt.Sprintf("Found %d rules in file: %q", len(found), bazelFile.RelPath)) for _, rule := range found { changedRule, ruleIssues := upgradeRule(ctx, mirrorUpload, rule, log) if len(ruleIssues) > 0 { @@ -108,11 +109,11 @@ func upgradeBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirror } if len(iss) > 0 { - log.Warnf("File %s has issues. Not saving!", bazelFile.RelPath) + log.Warn(fmt.Sprintf("File %s has issues. Not saving!", bazelFile.RelPath)) return iss, nil } if !changed { - log.Debugf("No changes to file: %s", bazelFile.RelPath) + log.Debug(fmt.Sprintf("No changes to file: %q", bazelFile.RelPath)) return iss, nil } if dryRun { @@ -120,10 +121,10 @@ func upgradeBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirror if err != nil { return iss, err } - log.Infof("Dry run: would save updated file %s with diff:\n%s", bazelFile.RelPath, diff) + log.Info(fmt.Sprintf("Dry run: would save updated file %s with diff:\n%s", bazelFile.RelPath, diff)) return iss, nil } - log.Infof("Saving updated file: %s", bazelFile.RelPath) + log.Info(fmt.Sprintf("Saving updated file: %s", bazelFile.RelPath)) if err := fileHelper.WriteFile(bazelFile, buildfile); err != nil { return iss, err } @@ -131,12 +132,12 @@ func upgradeBazelFile(ctx context.Context, fileHelper *bazelfiles.Helper, mirror return iss, nil } -func upgradeRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *logger.Logger) (changed bool, iss []error) { - log.Debugf("Upgrading rule: %s", rule.Name()) +func upgradeRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.Rule, log *slog.Logger) (changed bool, iss []error) { + log.Debug(fmt.Sprintf("Upgrading rule: %q", rule.Name())) upstreamURLs, err := rules.UpstreamURLs(rule) if errors.Is(err, rules.ErrNoUpstreamURL) { - log.Debugf("Rule has no upstream URL. Skipping.") + log.Debug("Rule has no upstream URL. Skipping.") return false, nil } else if err != nil { iss = append(iss, err) @@ -152,7 +153,7 @@ func upgradeRule(ctx context.Context, mirrorUpload mirrorUploader, rule *build.R existingHash, err := rules.GetHash(rule) if err == nil && learnedHash == existingHash { - log.Debugf("Rule already upgraded. Skipping.") + log.Debug("Rule already upgraded. Skipping.") return false, nil } @@ -177,7 +178,7 @@ type upgradeFlags struct { region string bucket string mirrorBaseURL string - logLevel zapcore.Level + logLevel slog.Level } func parseUpgradeFlags(cmd *cobra.Command) (upgradeFlags, error) { @@ -193,9 +194,9 @@ func parseUpgradeFlags(cmd *cobra.Command) (upgradeFlags, error) { if err != nil { return upgradeFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } region, err := cmd.Flags().GetString("region") if err != nil { diff --git a/hack/cli-k8s-compatibility/BUILD.bazel b/hack/cli-k8s-compatibility/BUILD.bazel index 8ea67b87d..4c94d507e 100644 --- a/hack/cli-k8s-compatibility/BUILD.bazel +++ b/hack/cli-k8s-compatibility/BUILD.bazel @@ -7,9 +7,9 @@ go_library( visibility = ["//visibility:private"], deps = [ "//internal/api/versionsapi", + "//internal/constants", "//internal/logger", "//internal/versions", - "@org_uber_go_zap//zapcore", ], ) diff --git a/hack/cli-k8s-compatibility/main.go b/hack/cli-k8s-compatibility/main.go index 1f58209a3..e1cb7c611 100644 --- a/hack/cli-k8s-compatibility/main.go +++ b/hack/cli-k8s-compatibility/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // cli-k8s-compatibility generates JSON output for a CLI version and its supported Kubernetes versions. @@ -10,11 +10,14 @@ package main import ( "context" "flag" + "fmt" + "log/slog" + "os" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/versions" - "go.uber.org/zap/zapcore" ) var ( @@ -24,22 +27,25 @@ var ( ) func main() { - log := logger.New(logger.PlainLog, zapcore.DebugLevel) + log := logger.NewTextLogger(slog.LevelDebug) ctx := context.Background() flag.Parse() if *refFlag == "" { - log.Fatalf("ref must be set") + log.Error("ref must be set") + os.Exit(1) } if *streamFlag == "" { - log.Fatalf("stream must be set") + log.Error("stream must be set") + os.Exit(1) } if *versionFlag == "" { - log.Fatalf("version must be set") + log.Error("version must be set") + os.Exit(1) } cliInfo := versionsapi.CLIInfo{ - Ref: *refFlag, + Ref: versionsapi.CanonicalizeRef(*refFlag), Stream: *streamFlag, Version: *versionFlag, Kubernetes: []string{}, @@ -49,17 +55,20 @@ func main() { cliInfo.Kubernetes = append(cliInfo.Kubernetes, v.ClusterVersion) } - c, cclose, err := versionsapi.NewClient(ctx, "eu-central-1", "cdn-constellation-backend", "E1H77EZTHC3NE4", false, log) + c, cclose, err := versionsapi.NewClient(ctx, "eu-central-1", "cdn-constellation-backend", constants.CDNDefaultDistributionID, false, log) if err != nil { - log.Fatalf("creating s3 client: %w", err) + log.Error(fmt.Sprintf("creating s3 client: %s", err)) + os.Exit(1) } defer func() { if err := cclose(ctx); err != nil { - log.Fatalf("invalidating cache: %w", err) + log.Error(fmt.Sprintf("invalidating cache: %s", err)) + os.Exit(1) } }() if err := c.UpdateCLIInfo(ctx, cliInfo); err != nil { - log.Fatalf("updating cli info: %w", err) + log.Error(fmt.Sprintf("updating cli info: %s", err)) + os.Exit(1) } } diff --git a/hack/clidocgen/main.go b/hack/clidocgen/main.go index b63b796cb..599e30d22 100644 --- a/hack/clidocgen/main.go +++ b/hack/clidocgen/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Clidocgen generates a Markdown page describing all CLI commands. diff --git a/hack/go.mod b/hack/go.mod deleted file mode 100644 index 04aad6a26..000000000 --- a/hack/go.mod +++ /dev/null @@ -1,316 +0,0 @@ -module github.com/edgelesssys/constellation/v2/hack - -go 1.21 - -replace ( - k8s.io/api v0.0.0 => k8s.io/api v0.28.2 - k8s.io/apiextensions-apiserver v0.0.0 => k8s.io/apiextensions-apiserver v0.27.2 - k8s.io/apimachinery v0.0.0 => k8s.io/apimachinery v0.27.2 - k8s.io/apiserver v0.0.0 => k8s.io/apiserver v0.27.2 - k8s.io/cli-runtime v0.0.0 => k8s.io/cli-runtime v0.27.2 - k8s.io/client-go v0.0.0 => k8s.io/client-go v0.27.2 - k8s.io/cloud-provider v0.0.0 => k8s.io/cloud-provider v0.27.2 - k8s.io/cluster-bootstrap v0.0.0 => k8s.io/cluster-bootstrap v0.27.2 - k8s.io/code-generator v0.0.0 => k8s.io/code-generator v0.27.2 - k8s.io/component-base v0.0.0 => k8s.io/component-base v0.27.2 - k8s.io/component-helpers v0.0.0 => k8s.io/component-helpers v0.27.2 - k8s.io/controller-manager v0.0.0 => k8s.io/controller-manager v0.27.2 - k8s.io/cri-api v0.0.0 => k8s.io/cri-api v0.27.2 - k8s.io/csi-translation-lib v0.0.0 => k8s.io/csi-translation-lib v0.27.2 - k8s.io/dynamic-resource-allocation v0.0.0 => k8s.io/dynamic-resource-allocation v0.27.2 - k8s.io/kube-aggregator v0.0.0 => k8s.io/kube-aggregator v0.27.2 - k8s.io/kube-controller-manager v0.0.0 => k8s.io/kube-controller-manager v0.27.2 - k8s.io/kube-proxy v0.0.0 => k8s.io/kube-proxy v0.27.2 - k8s.io/kube-scheduler v0.0.0 => k8s.io/kube-scheduler v0.27.2 - k8s.io/kubectl v0.0.0 => k8s.io/kubectl v0.27.2 - k8s.io/kubelet v0.0.0 => k8s.io/kubelet v0.27.2 - k8s.io/legacy-cloud-providers v0.0.0 => k8s.io/legacy-cloud-providers v0.27.2 - k8s.io/metrics v0.0.0 => k8s.io/metrics v0.27.2 - k8s.io/mount-utils v0.0.0 => k8s.io/mount-utils v0.27.2 - k8s.io/pod-security-admission v0.0.0 => k8s.io/pod-security-admission v0.27.2 - k8s.io/sample-apiserver v0.0.0 => k8s.io/sample-apiserver v0.27.2 -) - -replace ( - github.com/edgelesssys/constellation/v2 => ./.. - github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api => ./../operators/constellation-node-operator/api -) - -require ( - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 - github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 - github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e - github.com/edgelesssys/constellation/v2 v2.6.0 - github.com/hexops/gotextdiff v1.0.3 - github.com/spf13/afero v1.10.0 - github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.4 - go.uber.org/goleak v1.3.0 - go.uber.org/zap v1.26.0 - golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - gopkg.in/yaml.v3 v3.0.1 - libvirt.org/go/libvirt v1.9004.0 -) - -require ( - cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights v1.1.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.29 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect - github.com/MakeNowJust/heredoc v1.0.0 // indirect - github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/Masterminds/sprig/v3 v3.2.3 // indirect - github.com/Masterminds/squirrel v1.5.4 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.0 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect - github.com/agext/levenshtein v1.2.1 // indirect - github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go v1.44.297 // indirect - github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect - github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.8 // indirect - github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect - github.com/aws/smithy-go v1.13.5 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver v3.5.1+incompatible // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chai2010/gettext-go v1.0.2 // indirect - github.com/cloudflare/circl v1.3.3 // indirect - github.com/containerd/containerd v1.7.6 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/cli v24.0.6+incompatible // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.7+incompatible // indirect - github.com/docker/docker-credential-helpers v0.7.0 // indirect - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-metrics v0.0.1 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api v0.0.0 // indirect - github.com/edgelesssys/go-azguestattestation v0.0.0-20230707101700-a683be600fcf // indirect - github.com/edgelesssys/go-tdx-qpl v0.0.0-20230530085549-fd2878a4dead // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/go-chi/chi v4.1.2+incompatible // indirect - github.com/go-errors/errors v1.4.2 // indirect - github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/analysis v0.21.4 // indirect - github.com/go-openapi/errors v0.20.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/loads v0.21.2 // indirect - github.com/go-openapi/runtime v0.26.0 // indirect - github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/strfmt v0.21.7 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-openapi/validate v0.22.1 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.1 // indirect - github.com/gobwas/glob v0.2.3 // indirect - github.com/gofrs/uuid v4.2.0+incompatible // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/certificate-transparency-go v1.1.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-attestation v0.5.0 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-containerregistry v0.15.2 // indirect - github.com/google/go-sev-guest v0.9.3 // indirect - github.com/google/go-tdx-guest v0.2.3-0.20231011100059-4cf02bed9d33 // indirect - github.com/google/go-tpm v0.9.0 // indirect - github.com/google/go-tpm-tools v0.4.2 // indirect - github.com/google/go-tspi v0.3.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/logger v1.1.1 // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.4.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect - github.com/gophercloud/gophercloud v1.5.0 // indirect - github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 // indirect - github.com/gorilla/mux v1.8.0 // indirect - github.com/gosuri/uitable v0.0.4 // indirect - github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 // indirect - github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/hc-install v0.6.1 // indirect - github.com/hashicorp/hcl/v2 v2.19.1 // indirect - github.com/hashicorp/terraform-exec v0.19.0 // indirect - github.com/hashicorp/terraform-json v0.18.0 // indirect - github.com/huandu/xstrings v1.4.0 // indirect - github.com/imdario/mergo v0.3.15 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jmoiron/sqlx v1.3.5 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect - github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/leodido/go-urn v1.2.4 // indirect - github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf // indirect - github.com/lib/pq v1.10.9 // indirect - github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect - github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/locker v1.0.1 // indirect - github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect - github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/rubenv/sql-migrate v1.5.2 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sassoftware/relic v7.2.1+incompatible // indirect - github.com/schollz/progressbar/v3 v3.13.1 // indirect - github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect - github.com/sergi/go-diff v1.3.1 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - github.com/siderolabs/talos/pkg/machinery v1.4.6 // indirect - github.com/sigstore/rekor v1.2.2 // indirect - github.com/sigstore/sigstore v1.7.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/theupdateframework/go-tuf v0.5.2 // indirect - github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect - github.com/transparency-dev/merkle v0.0.2 // indirect - github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/xlab/treeprint v1.2.0 // indirect - github.com/zclconf/go-cty v1.14.1 // indirect - go.mongodb.org/mongo-driver v1.11.3 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect - go.opentelemetry.io/otel/trace v1.14.0 // indirect - go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.14.0 // indirect - google.golang.org/api v0.148.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - helm.sh/helm v2.17.0+incompatible // indirect - helm.sh/helm/v3 v3.13.1 // indirect - k8s.io/api v0.28.2 // indirect - k8s.io/apiextensions-apiserver v0.28.2 // indirect - k8s.io/apimachinery v0.28.2 // indirect - k8s.io/apiserver v0.28.2 // indirect - k8s.io/cli-runtime v0.28.2 // indirect - k8s.io/client-go v0.28.2 // indirect - k8s.io/cluster-bootstrap v0.27.3 // indirect - k8s.io/component-base v0.28.2 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/kubectl v0.28.2 // indirect - k8s.io/kubernetes v1.27.8 // indirect - k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect - oras.land/oras-go v1.2.4 // indirect - sigs.k8s.io/controller-runtime v0.15.0 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect -) diff --git a/hack/go.sum b/hack/go.sum deleted file mode 100644 index aa08f25d5..000000000 --- a/hack/go.sum +++ /dev/null @@ -1,1388 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ= -code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230618160516-e936619f9f18 h1:rd389Q26LMy03gG4anandGFC2LW/xvjga5GezeeaxQk= -github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230618160516-e936619f9f18/go.mod h1:fgJuSBrJP5qZtKqaMJE0hmhS2tmRH+44IkfZvjtaf1M= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights v1.1.1 h1:hBrFatNIiVAwDb5GzMLjpkQ6l2/waFSvBWMBWZRH8WI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights v1.1.1/go.mod h1:uxknLoFj+nBXpfGngz0B4ciNur04Y0EX4AREpy2GIvk= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 h1:Sg/D8VuUQ+bw+FOYJF+xRKcwizCOP13HL0Se8pWNBzE= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0/go.mod h1:Kyqzdqq0XDoCm+o9aZ25wZBmBUBzPBzPAj1R5rYsT6I= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2/go.mod h1:FbdwsQ2EzwvXxOPcMFYO8ogEc9uMMIj3YkmCdXdAFmk= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.0.0 h1:pqCyNi/Paz03SbWRmGlb5WBzK14aOXVuSJuOTWzOM5M= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.0.0/go.mod h1:bCUhQ1sbQHAG4nm1SqWwLlnKnRVT2e6Lu0cij7OzliM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= -github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= -github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= -github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= -github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= -github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= -github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= -github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= -github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= -github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= -github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= -github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= -github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.44.297 h1:uL4EV0gQxotQVYegIoBqK079328MOJqgG95daFYSkAM= -github.com/aws/aws-sdk-go v1.44.297/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= -github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= -github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= -github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71 h1:SAB1UAVaf6nGCu3zyIrV+VWsendXrms1GqtW4zBotKA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.71/go.mod h1:ZNo5H4PR3/fwsXYqb+Ld5YAfvHcYCbltaTTtSay4l2o= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.8 h1:loRDtQ0vT0+JCB0hQBCfv95tttEzJ1rqSaTDy5cpy0A= -github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.8/go.mod h1:YTd4wGn2beCF9wkSTpEcupk79zDFYJk2Ca76B8YyvJg= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 h1:P4dyjm49F2kKws0FpouBC6fjVImACXKt752+CWa01lM= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0/go.mod h1:tIctCeX9IbzsUTKHt53SVEcgyfxV2ElxJeEB+QUbc4M= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 h1:lEmQ1XSD9qLk+NZXbgvLJI/IiTz7OIR2TYUTFH25EI4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= -github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e h1:XmPu4mXICgdGnC5dXGjUGbwUD/kUmS0l5Aop3LaevBM= -github.com/bazelbuild/buildtools v0.0.0-20230317132445-9c3c1fc0106e/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= -github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= -github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= -github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= -github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= -github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= -github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 h1:vU+EP9ZuFUCYE0NYLwTSob+3LNEJATzNfP/DC7SWGWI= -github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= -github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= -github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/edgelesssys/go-azguestattestation v0.0.0-20230707101700-a683be600fcf h1:1iKB7b+i7svWC0aKXwggi+kHf0K57g8r9hN4VOpJYYg= -github.com/edgelesssys/go-azguestattestation v0.0.0-20230707101700-a683be600fcf/go.mod h1:T8Rv3qrCpUJZbKq49OA9tcC1ZbRkGtDxiafsj++LYIE= -github.com/edgelesssys/go-tdx-qpl v0.0.0-20230530085549-fd2878a4dead h1:Q2TI34V/NCLGQQkdc0/KmPx/7ix9YnGDUQDT+gqvDw0= -github.com/edgelesssys/go-tdx-qpl v0.0.0-20230530085549-fd2878a4dead/go.mod h1:IC72qyykUIWl0ZmSk53L4xbLCFDBEGZVaujUmPQOEyw= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= -github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg= -github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01/go.mod h1:ypD5nozFk9vcGw1ATYefw6jHe/jZP++Z15/+VTMcWhc= -github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8= -github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52/go.mod h1:yIquW87NGRw1FU5p5lEkpnt/QxoH5uPAOUlOVkAUuMg= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= -github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= -github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= -github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= -github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= -github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= -github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/analysis v0.21.4 h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc= -github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.4 h1:unTcVm6PispJsMECE3zWgvG4xTiKda1LIR5rCRWLG6M= -github.com/go-openapi/errors v0.20.4/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro= -github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= -github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= -github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= -github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= -github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= -github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+jU0zvx4AqHGnv4k= -github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= -github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= -github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0= -github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= -github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= -github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= -github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= -github.com/google/certificate-transparency-go v1.1.4 h1:hCyXHDbtqlr/lMXU0D4WgbalXL0Zk4dSWWMbPV8VrqY= -github.com/google/certificate-transparency-go v1.1.4/go.mod h1:D6lvbfwckhNrbM9WVl1EVeMOyzC19mpIjMOI4nxBHtQ= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-attestation v0.5.0 h1:jXtAWT2sw2Yu8mYU0BC7FDidR+ngxFPSE+pl6IUu3/0= -github.com/google/go-attestation v0.5.0/go.mod h1:0Tik9y3rzV649Jcr7evbljQHQAsIlJucyqQjYDBqktU= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= -github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= -github.com/google/go-sev-guest v0.9.3 h1:GOJ+EipURdeWFl/YYdgcCxyPeMgQUWlI056iFkBD8UU= -github.com/google/go-sev-guest v0.9.3/go.mod h1:hc1R4R6f8+NcJwITs0L90fYWTsBpd1Ix+Gur15sqHDs= -github.com/google/go-tdx-guest v0.2.3-0.20231011100059-4cf02bed9d33 h1:lRlUusuieEuqljjihCXb+Mr73VNitOYPJYWXzJKtBWs= -github.com/google/go-tdx-guest v0.2.3-0.20231011100059-4cf02bed9d33/go.mod h1:84ut3oago/BqPXD4ppiGXdkZNW3WFPkcyAO4my2hXdY= -github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= -github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= -github.com/google/go-tpm-tools v0.4.2 h1:iyaCPKt2N5Rd0yz0G8ANa022SgCNZkMpp+db6QELtvI= -github.com/google/go-tpm-tools v0.4.2/go.mod h1:fGUDZu4tw3V4hUVuFHmiYgRd0c58/IXivn9v3Ea/ck4= -github.com/google/go-tspi v0.3.0 h1:ADtq8RKfP+jrTyIWIZDIYcKOMecRqNJFOew2IT0Inus= -github.com/google/go-tspi v0.3.0/go.mod h1:xfMGI3G0PhxCdNVcYr1C4C+EizojDg/TXuX5by8CiHI= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ= -github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gophercloud/gophercloud v1.3.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= -github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= -github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= -github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56 h1:sH7xkTfYzxIEgzq1tDHIMKRh1vThOEOGNsettdEeLbE= -github.com/gophercloud/utils v0.0.0-20231010081019-80377eca5d56/go.mod h1:VSalo4adEk+3sNkmVJLnhHoOyOYYS8sTWLG4mv5BKto= -github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= -github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= -github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hc-install v0.6.1 h1:IGxShH7AVhPaSuSJpKtVi/EFORNjO+OYVJJrAtGG2mY= -github.com/hashicorp/hc-install v0.6.1/go.mod h1:0fW3jpg+wraYSnFDJ6Rlie3RvLf1bIqVIkzoon4KoVE= -github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI= -github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= -github.com/hashicorp/terraform-exec v0.19.0 h1:FpqZ6n50Tk95mItTSS9BjeOVUb4eg81SpgVtZNNtFSM= -github.com/hashicorp/terraform-exec v0.19.0/go.mod h1:tbxUpe3JKruE9Cuf65mycSIT8KiNPZ0FkuTE3H4urQg= -github.com/hashicorp/terraform-json v0.18.0 h1:pCjgJEqqDESv4y0Tzdqfxr/edOIGkjs8keY42xfNBwU= -github.com/hashicorp/terraform-json v0.18.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/honeycombio/beeline-go v1.10.0 h1:cUDe555oqvw8oD76BQJ8alk7FP0JZ/M/zXpNvOEDLDc= -github.com/honeycombio/beeline-go v1.10.0/go.mod h1:Zz5WMeQCJzFt2Mvf8t6HC1X8RLskLVR/e8rvcmXB1G8= -github.com/honeycombio/libhoney-go v1.16.0 h1:kPpqoz6vbOzgp7jC6SR7SkNj7rua7rgxvznI6M3KdHc= -github.com/honeycombio/libhoney-go v1.16.0/go.mod h1:izP4fbREuZ3vqC4HlCAmPrcPT9gxyxejRjGtCYpmBn0= -github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= -github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= -github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b h1:ZGiXF8sz7PDk6RgkP+A/SFfUD0ZR/AgG6SpRNEDKZy8= -github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b/go.mod h1:hQmNrgofl+IY/8L+n20H6E6PWBBTokdsv+q49j0QhsU= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548 h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4= -github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= -github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf h1:ndns1qx/5dL43g16EQkPV/i8+b3l5bYQwLeoSBe7tS8= -github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf/go.mod h1:aGkAgvWY/IUcVFfuly53REpfv5edu25oij+qHRFaraA= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= -github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= -github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= -github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= -github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= -github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= -github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= -github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= -github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A= -github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk= -github.com/sassoftware/relic/v7 v7.5.5 h1:2ZUM6ovo3STCAp0hZnO9nQY9lOB8OyfneeYIi4YUxMU= -github.com/sassoftware/relic/v7 v7.5.5/go.mod h1:NxwtWxWxlUa9as2qZi635Ye6bBT/tGnMALLq7dSfOOU= -github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= -github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= -github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA= -github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/siderolabs/talos/pkg/machinery v1.4.6 h1:SX7Q6FxTDyX2hxugMgIqyivXWzemgMhHj3AlDbxjuFw= -github.com/siderolabs/talos/pkg/machinery v1.4.6/go.mod h1:l+3Akw0tz/k/IMbqQS+cdQaqVSitIIJf04+N9ahGn+E= -github.com/sigstore/rekor v1.2.2 h1:5JK/zKZvcQpL/jBmHvmFj3YbpDMBQnJQ6ygp8xdF3bY= -github.com/sigstore/rekor v1.2.2/go.mod h1:FGnWBGWzeNceJnp0x9eDFd41mI8aQqCjj+Zp0IEs0Qg= -github.com/sigstore/sigstore v1.7.1 h1:fCATemikcBK0cG4+NcM940MfoIgmioY1vC6E66hXxks= -github.com/sigstore/sigstore v1.7.1/go.mod h1:0PmMzfJP2Y9+lugD0wer4e7TihR5tM7NcIs3bQNk5xg= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= -github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= -github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA= -github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= -github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= -github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= -github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= -github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= -github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= -github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= -github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= -github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810 h1:X6ps8XHfpQjw8dUStzlMi2ybiKQ2Fmdw7UM+TinwvyM= -github.com/vtolstov/go-ioctl v0.0.0-20151206205506-6be9cced4810/go.mod h1:dF0BBJ2YrV1+2eAIyEI+KeSidgA6HqoIP1u5XTlMq/o= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= -github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= -github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/zalando/go-keyring v0.2.2 h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4= -github.com/zalando/go-keyring v0.2.2/go.mod h1:sI3evg9Wvpw3+n4SqplGSJUMwtDeROfD4nsFz4z9PG0= -github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA= -github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y= -go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= -go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= -go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= -golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs= -google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= -gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= -gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -helm.sh/helm v2.17.0+incompatible h1:cSe3FaQOpRWLDXvTObQNj0P7WI98IG5yloU6tQVls2k= -helm.sh/helm v2.17.0+incompatible/go.mod h1:0Xbc6ErzwWH9qC55X1+hE3ZwhM3atbhCm/NbFZw5i+4= -helm.sh/helm/v3 v3.13.1 h1:DG+XLGzBJeZvMLlMbm6bPDLV1dGaVW9eZsDoUd1/LM0= -helm.sh/helm/v3 v3.13.1/go.mod h1:TdQRMiq46CSWcc68Hb0uVhvAWusaN90YwAV54cz6JzU= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= -k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/apiserver v0.28.2 h1:rBeYkLvF94Nku9XfXyUIirsVzCzJBs6jMn3NWeHieyI= -k8s.io/apiserver v0.28.2/go.mod h1:f7D5e8wH8MWcKD7azq6Csw9UN+CjdtXIVQUyUhrtb+E= -k8s.io/cli-runtime v0.28.2 h1:64meB2fDj10/ThIMEJLO29a1oujSm0GQmKzh1RtA/uk= -k8s.io/cli-runtime v0.28.2/go.mod h1:bTpGOvpdsPtDKoyfG4EG041WIyFZLV9qq4rPlkyYfDA= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= -k8s.io/cluster-bootstrap v0.27.3 h1:yk1XIWt/mbMgNHFdxd0HyVPq/rnJK7BS3oXj24gHClU= -k8s.io/cluster-bootstrap v0.27.3/go.mod h1:4/bxgDkpV7XPapJS1585P/nvy3FdBIoFssK4Z5oztrc= -k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= -k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM= -k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64= -k8s.io/kubernetes v1.27.8 h1:K848lTo/D0jvrxUlTvw4nNADixbhXLHgKNDP/KlFGy8= -k8s.io/kubernetes v1.27.8/go.mod h1:PUXXrx0IhAi+kI9BMDqNJHUnLndVv9W0DkriqyjuJOs= -k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= -k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -libvirt.org/go/libvirt v1.9004.0 h1:u+CHhs2OhVmu0MWzBDrlbLzQ5QB3ZfWtfT+lD3EaUIs= -libvirt.org/go/libvirt v1.9004.0/go.mod h1:1WiFE8EjZfq+FCVog+rvr1yatKbKZ9FaFMZgEqxEJqQ= -oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= -oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= -sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= -sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= -sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE= -software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ= diff --git a/hack/gocoverage/main.go b/hack/gocoverage/main.go index d119a5bf5..c6b755cde 100644 --- a/hack/gocoverage/main.go +++ b/hack/gocoverage/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -139,25 +139,25 @@ func parseTestLine(line string) (string, packageCoverage, error) { return pkg, coverage, nil } -func diff(old, new, touched string) error { - oldf, err := os.Open(old) +func diff(a, b, touched string) error { + af, err := os.Open(a) if err != nil { return err } - defer oldf.Close() - var oldRep, newReport report - if err := json.NewDecoder(oldf).Decode(&oldRep); err != nil { + defer af.Close() + var aReport, bReport report + if err := json.NewDecoder(af).Decode(&aReport); err != nil { return err } - newf, err := os.Open(new) + bf, err := os.Open(b) if err != nil { return err } - defer newf.Close() - if err := json.NewDecoder(newf).Decode(&newReport); err != nil { + defer bf.Close() + if err := json.NewDecoder(bf).Decode(&bReport); err != nil { return err } - diffs, err := diffCoverage(oldRep, newReport) + diffs, err := diffCoverage(aReport, bReport) if err != nil { return err } @@ -170,17 +170,17 @@ type coverageDiff struct { new *packageCoverage } -func diffCoverage(old, new report) (map[string]coverageDiff, error) { +func diffCoverage(a, b report) (map[string]coverageDiff, error) { allPkgs := make(map[string]struct{}) - for pkg := range old.Coverage { + for pkg := range a.Coverage { allPkgs[pkg] = struct{}{} } - for pkg := range new.Coverage { + for pkg := range b.Coverage { allPkgs[pkg] = struct{}{} } diffs := make(map[string]coverageDiff) for pkg := range allPkgs { - diffs[pkg] = coverageDiff{old: old.Coverage[pkg], new: new.Coverage[pkg]} + diffs[pkg] = coverageDiff{old: a.Coverage[pkg], new: b.Coverage[pkg]} if diffs[pkg].old == nil && diffs[pkg].new == nil { return nil, fmt.Errorf("both old and new coverage are nil for pkg %s", pkg) } diff --git a/hack/gocoverage/main_test.go b/hack/gocoverage/main_test.go index 18c9fe139..267aa1522 100644 --- a/hack/gocoverage/main_test.go +++ b/hack/gocoverage/main_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -204,25 +204,25 @@ ok github.com/edgelesssys/constellation/v2/hack/oci-pin/internal/extract (cach ok github.com/edgelesssys/constellation/v2/hack/oci-pin/internal/inject (cached) coverage: 100.0% of statements ok github.com/edgelesssys/constellation/v2/hack/oci-pin/internal/sums (cached) coverage: 98.9% of statements ok github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/server (cached) coverage: 60.7% of statements -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2 [no test files] -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api [no test files] -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/controllers (cached) coverage: 30.6% of statements -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/fake/client [no test files] -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/aws/client (cached) coverage: 78.4% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/azure/client (cached) coverage: 89.3% of statements -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/constants [no test files] -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/gcp/client (cached) coverage: 80.8% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/controlplane (cached) coverage: 100.0% of statements -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/upgrade [no test files] -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/deploy (cached) coverage: 76.6% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/etcd (cached) coverage: 65.8% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/executor (cached) coverage: 93.2% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/node (cached) coverage: 100.0% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/patch (cached) coverage: 100.0% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/poller (cached) coverage: 91.4% of statements -ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/sgreconciler (cached) coverage: 34.9% of statements -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api [no test files] -? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1 [no test files] +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator [no test files] +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/cloud/api [no test files] +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/controllers (cached) coverage: 30.6% of statements +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/cloud/fake/client [no test files] +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/cloud/aws/client (cached) coverage: 78.4% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/cloud/azure/client (cached) coverage: 89.3% of statements +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants [no test files] +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/cloud/gcp/client (cached) coverage: 80.8% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/controlplane (cached) coverage: 100.0% of statements +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/upgrade [no test files] +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/deploy (cached) coverage: 76.6% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/etcd (cached) coverage: 65.8% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/executor (cached) coverage: 93.2% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/node (cached) coverage: 100.0% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/patch (cached) coverage: 100.0% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/poller (cached) coverage: 91.4% of statements +ok github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/sgreconciler (cached) coverage: 34.9% of statements +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api [no test files] +? github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1 [no test files] ` const ( diff --git a/hack/image-fetch/main.go b/hack/image-fetch/main.go index 7a88801a8..4124a9f54 100644 --- a/hack/image-fetch/main.go +++ b/hack/image-fetch/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/hack/image-fetch/main_test.go b/hack/image-fetch/main_test.go index fa73af23e..30fe94af4 100644 --- a/hack/image-fetch/main_test.go +++ b/hack/image-fetch/main_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main diff --git a/hack/logcollector/cmd/root.go b/hack/logcollector/cmd/root.go index 9af040c49..a6f8f1501 100644 --- a/hack/logcollector/cmd/root.go +++ b/hack/logcollector/cmd/root.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/hack/logcollector/cmd/template.go b/hack/logcollector/cmd/template.go index 8776a52df..1812981ae 100644 --- a/hack/logcollector/cmd/template.go +++ b/hack/logcollector/cmd/template.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/hack/logcollector/fields/fields.go b/hack/logcollector/fields/fields.go index d905f2b66..4443b3415 100644 --- a/hack/logcollector/fields/fields.go +++ b/hack/logcollector/fields/fields.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package fields diff --git a/hack/logcollector/internal/filebeat.go b/hack/logcollector/internal/filebeat.go index bb7fd8493..d46812e37 100644 --- a/hack/logcollector/internal/filebeat.go +++ b/hack/logcollector/internal/filebeat.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package internal diff --git a/hack/logcollector/internal/logstash.go b/hack/logcollector/internal/logstash.go index ea03365e6..808ed4d32 100644 --- a/hack/logcollector/internal/logstash.go +++ b/hack/logcollector/internal/logstash.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package internal diff --git a/hack/logcollector/internal/metricbeat.go b/hack/logcollector/internal/metricbeat.go index 603f2dcd4..0d71ea125 100644 --- a/hack/logcollector/internal/metricbeat.go +++ b/hack/logcollector/internal/metricbeat.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package internal diff --git a/hack/logcollector/internal/prepare.go b/hack/logcollector/internal/prepare.go index 8f6408ac4..acfec0114 100644 --- a/hack/logcollector/internal/prepare.go +++ b/hack/logcollector/internal/prepare.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package internal diff --git a/hack/logcollector/main.go b/hack/logcollector/main.go index f4f6aaf96..3535e59dc 100644 --- a/hack/logcollector/main.go +++ b/hack/logcollector/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main diff --git a/hack/oci-pin/BUILD.bazel b/hack/oci-pin/BUILD.bazel index 9a3603791..15217ba6f 100644 --- a/hack/oci-pin/BUILD.bazel +++ b/hack/oci-pin/BUILD.bazel @@ -16,7 +16,6 @@ go_library( "//hack/oci-pin/internal/sums", "//internal/logger", "@com_github_spf13_cobra//:cobra", - "@org_uber_go_zap//zapcore", ], ) diff --git a/hack/oci-pin/codegen.go b/hack/oci-pin/codegen.go index 4c8f9fafc..b733c834b 100644 --- a/hack/oci-pin/codegen.go +++ b/hack/oci-pin/codegen.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -8,6 +8,7 @@ package main import ( "fmt" "io" + "log/slog" "os" "path/filepath" "strings" @@ -16,7 +17,6 @@ import ( "github.com/edgelesssys/constellation/v2/hack/oci-pin/internal/inject" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newCodegenCmd() *cobra.Command { @@ -44,15 +44,15 @@ func runCodegen(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "identifier", flags.identifier, "imageRepoTag", flags.imageRepoTag, "ociPath", flags.ociPath, "pkg", flags.pkg) registry, prefix, name, tag, err := splitRepoTag(flags.imageRepoTag) if err != nil { return fmt.Errorf("splitting OCI image reference %q: %w", flags.imageRepoTag, err) } - log.Debugf("Generating Go code for OCI image %s.", name) + log.Debug(fmt.Sprintf("Generating Go code for OCI image %q.", name)) ociIndexPath := filepath.Join(flags.ociPath, "index.json") index, err := os.Open(ociIndexPath) @@ -78,7 +78,7 @@ func runCodegen(cmd *cobra.Command, _ []string) error { return err } - log.Debugf("OCI image digest: %s", digest) + log.Debug(fmt.Sprintf("OCI image digest: %q", digest)) if err := inject.Render(out, inject.PinningValues{ Package: flags.pkg, @@ -92,7 +92,7 @@ func runCodegen(cmd *cobra.Command, _ []string) error { return fmt.Errorf("rendering Go code: %w", err) } - log.Debugf("Go code created at %q 🤖", flags.output) + log.Debug(fmt.Sprintf("Go code created at %q 🤖", flags.output)) return nil } @@ -102,7 +102,7 @@ type codegenFlags struct { pkg string identifier string imageRepoTag string - logLevel zapcore.Level + logLevel slog.Level } func parseCodegenFlags(cmd *cobra.Command) (codegenFlags, error) { @@ -137,9 +137,9 @@ func parseCodegenFlags(cmd *cobra.Command) (codegenFlags, error) { if err != nil { return codegenFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return codegenFlags{ diff --git a/hack/oci-pin/internal/extract/extract.go b/hack/oci-pin/internal/extract/extract.go index 5b76e954d..ae37c9501 100644 --- a/hack/oci-pin/internal/extract/extract.go +++ b/hack/oci-pin/internal/extract/extract.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package extract diff --git a/hack/oci-pin/internal/extract/extract_test.go b/hack/oci-pin/internal/extract/extract_test.go index cf3dddd93..893a0de20 100644 --- a/hack/oci-pin/internal/extract/extract_test.go +++ b/hack/oci-pin/internal/extract/extract_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package extract diff --git a/hack/oci-pin/internal/inject/inject.go b/hack/oci-pin/internal/inject/inject.go index f70ef4952..7175d7e9f 100644 --- a/hack/oci-pin/internal/inject/inject.go +++ b/hack/oci-pin/internal/inject/inject.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // inject renders Go source files with injected pinning values. diff --git a/hack/oci-pin/internal/inject/inject_test.go b/hack/oci-pin/internal/inject/inject_test.go index 9d209be11..e33cf1887 100644 --- a/hack/oci-pin/internal/inject/inject_test.go +++ b/hack/oci-pin/internal/inject/inject_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package inject diff --git a/hack/oci-pin/internal/sums/sums.go b/hack/oci-pin/internal/sums/sums.go index 48545e9e3..b6acdd426 100644 --- a/hack/oci-pin/internal/sums/sums.go +++ b/hack/oci-pin/internal/sums/sums.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // sums creates and combines sha256sums files. diff --git a/hack/oci-pin/internal/sums/sums_test.go b/hack/oci-pin/internal/sums/sums_test.go index 4796bd028..3bc61fe79 100644 --- a/hack/oci-pin/internal/sums/sums_test.go +++ b/hack/oci-pin/internal/sums/sums_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package sums diff --git a/hack/oci-pin/merge.go b/hack/oci-pin/merge.go index 081ea6a6f..d13b84c3c 100644 --- a/hack/oci-pin/merge.go +++ b/hack/oci-pin/merge.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -8,12 +8,12 @@ package main import ( "fmt" "io" + "log/slog" "os" "github.com/edgelesssys/constellation/v2/hack/oci-pin/internal/sums" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newMergeCmd() *cobra.Command { @@ -35,10 +35,10 @@ func runMerge(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "inputs", flags.inputs, "output", flags.output, "logLevel", flags.logLevel) - log.Debugf("Merging sum file from %q into %q.", flags.inputs, flags.output) + log.Debug(fmt.Sprintf("Merging sum file from %q into %q.", flags.inputs, flags.output)) var out io.Writer if flags.output == "-" { @@ -61,7 +61,7 @@ func runMerge(cmd *cobra.Command, _ []string) error { return fmt.Errorf("creating merged sum file: %w", err) } - log.Debugf("Sum file created at %q 🤖", flags.output) + log.Debug(fmt.Sprintf("Sum file created at %q 🤖", flags.output)) return nil } @@ -93,7 +93,7 @@ func parseInput(input string) ([]sums.PinnedImageReference, error) { type mergeFlags struct { inputs []string output string - logLevel zapcore.Level + logLevel slog.Level } func parseMergeFlags(cmd *cobra.Command) (mergeFlags, error) { @@ -109,9 +109,9 @@ func parseMergeFlags(cmd *cobra.Command) (mergeFlags, error) { if err != nil { return mergeFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return mergeFlags{ diff --git a/hack/oci-pin/oci-pin.go b/hack/oci-pin/oci-pin.go index 5b3206c15..1c68e42d3 100644 --- a/hack/oci-pin/oci-pin.go +++ b/hack/oci-pin/oci-pin.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // oci-pin generates Go code and shasum files for OCI images. diff --git a/hack/oci-pin/sum.go b/hack/oci-pin/sum.go index 2268ae358..ba77b727b 100644 --- a/hack/oci-pin/sum.go +++ b/hack/oci-pin/sum.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -8,6 +8,7 @@ package main import ( "fmt" "io" + "log/slog" "os" "path/filepath" "strings" @@ -16,7 +17,6 @@ import ( "github.com/edgelesssys/constellation/v2/hack/oci-pin/internal/sums" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newSumCmd() *cobra.Command { @@ -41,15 +41,15 @@ func runSum(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "imageRepoTag", flags.imageRepoTag, "ociPath", flags.ociPath) registry, prefix, name, tag, err := splitRepoTag(flags.imageRepoTag) if err != nil { return fmt.Errorf("splitting repo tag: %w", err) } - log.Debugf("Generating sum file for OCI image %s.", name) + log.Debug(fmt.Sprintf("Generating sum file for OCI image %q.", name)) ociIndexPath := filepath.Join(flags.ociPath, "index.json") index, err := os.Open(ociIndexPath) @@ -75,7 +75,7 @@ func runSum(cmd *cobra.Command, _ []string) error { return fmt.Errorf("extracting OCI image digest: %w", err) } - log.Debugf("OCI image digest: %s", digest) + log.Debug(fmt.Sprintf("OCI image digest: %q", digest)) refs := []sums.PinnedImageReference{ { @@ -91,7 +91,7 @@ func runSum(cmd *cobra.Command, _ []string) error { return fmt.Errorf("creating sum file: %w", err) } - log.Debugf("Sum file created at %q 🤖", flags.output) + log.Debug(fmt.Sprintf("Sum file created at %q 🤖", flags.output)) return nil } @@ -99,7 +99,7 @@ type sumFlags struct { ociPath string output string imageRepoTag string - logLevel zapcore.Level + logLevel slog.Level } func parseSumFlags(cmd *cobra.Command) (sumFlags, error) { @@ -126,9 +126,9 @@ func parseSumFlags(cmd *cobra.Command) (sumFlags, error) { if err != nil { return sumFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return sumFlags{ diff --git a/hack/qemu-metadata-api/BUILD.bazel b/hack/qemu-metadata-api/BUILD.bazel index 60e4f80ab..2cfbeb83a 100644 --- a/hack/qemu-metadata-api/BUILD.bazel +++ b/hack/qemu-metadata-api/BUILD.bazel @@ -13,12 +13,11 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api", visibility = ["//visibility:private"], deps = [ + "//hack/qemu-metadata-api/dhcp/dnsmasq", + "//hack/qemu-metadata-api/dhcp/virtwrapper", "//hack/qemu-metadata-api/server", - "//hack/qemu-metadata-api/virtwrapper", "//internal/logger", "@org_libvirt_go_libvirt//:libvirt", - "@org_uber_go_zap//:zap", - "@org_uber_go_zap//zapcore", ], ) diff --git a/hack/qemu-metadata-api/dhcp/BUILD.bazel b/hack/qemu-metadata-api/dhcp/BUILD.bazel new file mode 100644 index 000000000..21ba5404c --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/BUILD.bazel @@ -0,0 +1,8 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "dhcp", + srcs = ["dhcp.go"], + importpath = "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp", + visibility = ["//visibility:public"], +) diff --git a/hack/qemu-metadata-api/virtwrapper/virtwrapper.go b/hack/qemu-metadata-api/dhcp/dhcp.go similarity index 50% rename from hack/qemu-metadata-api/virtwrapper/virtwrapper.go rename to hack/qemu-metadata-api/dhcp/dhcp.go index 4e5fb6732..a6e3aa08a 100644 --- a/hack/qemu-metadata-api/virtwrapper/virtwrapper.go +++ b/hack/qemu-metadata-api/dhcp/dhcp.go @@ -1,12 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ -package virtwrapper +package dhcp -// NetworkDHCPLease abstracts a libvirt DHCP lease. +// NetworkDHCPLease abstracts a DHCP lease. type NetworkDHCPLease struct { IPaddr string Hostname string diff --git a/hack/qemu-metadata-api/dhcp/dnsmasq/BUILD.bazel b/hack/qemu-metadata-api/dhcp/dnsmasq/BUILD.bazel new file mode 100644 index 000000000..ab5bbd249 --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/dnsmasq/BUILD.bazel @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "dnsmasq", + srcs = ["dnsmasq.go"], + importpath = "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp/dnsmasq", + visibility = ["//visibility:public"], + deps = [ + "//hack/qemu-metadata-api/dhcp", + "@com_github_spf13_afero//:afero", + ], +) + +go_test( + name = "dnsmasq_test", + srcs = ["dnsmasq_test.go"], + embed = [":dnsmasq"], + deps = [ + "@com_github_spf13_afero//:afero", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/hack/qemu-metadata-api/dhcp/dnsmasq/dnsmasq.go b/hack/qemu-metadata-api/dhcp/dnsmasq/dnsmasq.go new file mode 100644 index 000000000..4374e074b --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/dnsmasq/dnsmasq.go @@ -0,0 +1,56 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package dnsmasq + +import ( + "bufio" + "strings" + + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp" + "github.com/spf13/afero" +) + +// DNSMasq is a DHCP lease getter for dnsmasq. +type DNSMasq struct { + leasesFileName string + fs *afero.Afero +} + +// New creates a new DNSMasq. +func New(leasesFileName string) *DNSMasq { + return &DNSMasq{ + leasesFileName: leasesFileName, + fs: &afero.Afero{Fs: afero.NewOsFs()}, + } +} + +// GetDHCPLeases returns the underlying DHCP leases. +func (d *DNSMasq) GetDHCPLeases() ([]dhcp.NetworkDHCPLease, error) { + file, err := d.fs.Open(d.leasesFileName) + if err != nil { + return nil, err + } + defer file.Close() + + // read file + var leases []dhcp.NetworkDHCPLease + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + // split by whitespace + fields := strings.Fields(line) + leases = append(leases, dhcp.NetworkDHCPLease{ + IPaddr: fields[2], + Hostname: fields[3], + }) + } + if err := scanner.Err(); err != nil { + return nil, err + } + + return leases, nil +} diff --git a/hack/qemu-metadata-api/dhcp/dnsmasq/dnsmasq_test.go b/hack/qemu-metadata-api/dhcp/dnsmasq/dnsmasq_test.go new file mode 100644 index 000000000..2f079a471 --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/dnsmasq/dnsmasq_test.go @@ -0,0 +1,40 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package dnsmasq + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetDHCPLeases(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + fs := afero.NewMemMapFs() + leasesFileName := "dnsmasq.leases" + leasesFile, err := fs.Create(leasesFileName) + require.NoError(err) + _, err = leasesFile.WriteString("1716219737 52:54:af:a1:98:9f 10.42.2.1 worker0 ff:c2:72:f6:09:00:02:00:00:ab:11:18:fc:48:85:40:3f:bc:41\n") + require.NoError(err) + _, err = leasesFile.WriteString("1716219735 52:54:7f:8f:ba:91 10.42.1.1 controlplane0 ff:c2:72:f6:09:00:02:00:00:ab:11:21:7c:b5:14:ec:43:b7:43\n") + require.NoError(err) + leasesFile.Close() + + d := DNSMasq{leasesFileName: leasesFileName, fs: &afero.Afero{Fs: fs}} + leases, err := d.GetDHCPLeases() + require.NoError(err) + + assert.Len(leases, 2) + assert.Equal("10.42.2.1", leases[0].IPaddr) + assert.Equal("worker0", leases[0].Hostname) + assert.Equal("10.42.1.1", leases[1].IPaddr) + assert.Equal("controlplane0", leases[1].Hostname) +} diff --git a/hack/qemu-metadata-api/virtwrapper/BUILD.bazel b/hack/qemu-metadata-api/dhcp/virtwrapper/BUILD.bazel similarity index 68% rename from hack/qemu-metadata-api/virtwrapper/BUILD.bazel rename to hack/qemu-metadata-api/dhcp/virtwrapper/BUILD.bazel index c1ad3b7c9..762c0b301 100644 --- a/hack/qemu-metadata-api/virtwrapper/BUILD.bazel +++ b/hack/qemu-metadata-api/dhcp/virtwrapper/BUILD.bazel @@ -7,7 +7,10 @@ go_library( "virtwrapper_cgo.go", "virtwrapper_cross.go", ], - importpath = "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/virtwrapper", + importpath = "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp/virtwrapper", visibility = ["//visibility:public"], - deps = ["@org_libvirt_go_libvirt//:libvirt"], + deps = [ + "//hack/qemu-metadata-api/dhcp", + "@org_libvirt_go_libvirt//:libvirt", + ], ) diff --git a/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper.go b/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper.go new file mode 100644 index 000000000..7a31ff337 --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper.go @@ -0,0 +1,7 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package virtwrapper diff --git a/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper_cgo.go b/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper_cgo.go new file mode 100644 index 000000000..b74c65110 --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper_cgo.go @@ -0,0 +1,60 @@ +//go:build cgo + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package virtwrapper + +import ( + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp" + + "libvirt.org/go/libvirt" +) + +// Connect wraps a libvirt connection. +type Connect struct { + conn *libvirt.Connect + networkName string +} + +// New creates a new libvirt Connct wrapper. +func New(conn *libvirt.Connect, networkName string) *Connect { + return &Connect{ + conn: conn, + networkName: networkName, + } +} + +// LookupNetworkByName looks up a network by name. +func (c *Connect) lookupNetworkByName(name string) (*libvirt.Network, error) { + net, err := c.conn.LookupNetworkByName(name) + if err != nil { + return nil, err + } + return net, nil +} + +// GetDHCPLeases returns the underlying DHCP leases. +func (c *Connect) GetDHCPLeases() ([]dhcp.NetworkDHCPLease, error) { + net, err := c.lookupNetworkByName(c.networkName) + if err != nil { + return nil, err + } + defer net.Free() + + leases, err := net.GetDHCPLeases() + if err != nil { + return nil, err + } + ret := make([]dhcp.NetworkDHCPLease, len(leases)) + for i, l := range leases { + ret[i] = dhcp.NetworkDHCPLease{ + IPaddr: l.IPaddr, + Hostname: l.Hostname, + } + } + return ret, nil +} diff --git a/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper_cross.go b/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper_cross.go new file mode 100644 index 000000000..0467ab907 --- /dev/null +++ b/hack/qemu-metadata-api/dhcp/virtwrapper/virtwrapper_cross.go @@ -0,0 +1,24 @@ +//go:build !cgo + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package virtwrapper + +import ( + "errors" + + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp" +) + +// Connect wraps a libvirt connection. +type Connect struct{} + +// GetDHCPLeases returns the underlying DHCP leases. +// This function errors if CGO is disabled. +func (n *Connect) GetDHCPLeases() ([]dhcp.NetworkDHCPLease, error) { + return nil, errors.New("using virtwrapper requires building with CGO") +} diff --git a/hack/qemu-metadata-api/main.go b/hack/qemu-metadata-api/main.go index bec42e2c7..0693b9cd4 100644 --- a/hack/qemu-metadata-api/main.go +++ b/hack/qemu-metadata-api/main.go @@ -3,39 +3,50 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main import ( "flag" + "log/slog" + "os" + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp/dnsmasq" + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp/virtwrapper" "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/server" - "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/virtwrapper" "github.com/edgelesssys/constellation/v2/internal/logger" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" "libvirt.org/go/libvirt" ) func main() { bindPort := flag.String("port", "8080", "Port to bind to") - targetNetwork := flag.String("network", "constellation-network", "Name of the network in QEMU to use") + targetNetwork := flag.String("network", "constellation-network", "Name of the network in libvirt") libvirtURI := flag.String("libvirt-uri", "qemu:///system", "URI of the libvirt connection") + leasesFileName := flag.String("dnsmasq-leases", "", "Path to the dnsmasq leases file") initSecretHash := flag.String("initsecrethash", "", "brcypt hash of the init secret") flag.Parse() - log := logger.New(logger.JSONLog, zapcore.InfoLevel) + log := logger.NewJSONLogger(slog.LevelInfo) - conn, err := libvirt.NewConnect(*libvirtURI) - if err != nil { - log.With(zap.Error(err)).Fatalf("Failed to connect to libvirt") + var leaseGetter server.LeaseGetter + if *leasesFileName == "" { + conn, err := libvirt.NewConnect(*libvirtURI) + if err != nil { + log.With(slog.Any("error", err)).Error("Failed to connect to libvirt") + os.Exit(1) + } + defer conn.Close() + leaseGetter = virtwrapper.New(conn, *targetNetwork) + } else { + log.Info("Using dnsmasq leases file") + leaseGetter = dnsmasq.New(*leasesFileName) } - defer conn.Close() - serv := server.New(log, *targetNetwork, *initSecretHash, &virtwrapper.Connect{Conn: conn}) + serv := server.New(log, *targetNetwork, *initSecretHash, leaseGetter) if err := serv.ListenAndServe(*bindPort); err != nil { - log.With(zap.Error(err)).Fatalf("Failed to serve") + log.With(slog.Any("error", err)).Error("Failed to serve") + os.Exit(1) } } diff --git a/hack/qemu-metadata-api/main_cross.go b/hack/qemu-metadata-api/main_cross.go index 18ac575ba..3398538fa 100644 --- a/hack/qemu-metadata-api/main_cross.go +++ b/hack/qemu-metadata-api/main_cross.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main diff --git a/hack/qemu-metadata-api/server/BUILD.bazel b/hack/qemu-metadata-api/server/BUILD.bazel index 9fe06769b..683ec416a 100644 --- a/hack/qemu-metadata-api/server/BUILD.bazel +++ b/hack/qemu-metadata-api/server/BUILD.bazel @@ -10,32 +10,25 @@ go_library( ], visibility = ["//visibility:public"], deps = [ - "//hack/qemu-metadata-api/virtwrapper", + "//hack/qemu-metadata-api/dhcp", "//internal/cloud/metadata", - "//internal/logger", "//internal/role", - "@org_uber_go_zap//:zap", ], ) go_test( name = "server_test", - srcs = [ - "server_cgo_test.go", - "server_cross_test.go", - "server_test.go", - ], + srcs = ["server_test.go"], embed = [":server"], # keep pure = "on", # keep race = "off", deps = [ - "//hack/qemu-metadata-api/virtwrapper", + "//hack/qemu-metadata-api/dhcp", "//internal/cloud/metadata", "//internal/logger", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@org_libvirt_go_libvirt//:libvirt", ], ) diff --git a/hack/qemu-metadata-api/server/server.go b/hack/qemu-metadata-api/server/server.go index c60bdae30..4394732b2 100644 --- a/hack/qemu-metadata-api/server/server.go +++ b/hack/qemu-metadata-api/server/server.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package server @@ -9,31 +9,29 @@ package server import ( "encoding/json" "fmt" - "io" + "log/slog" "net" "net/http" "strings" - "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/virtwrapper" + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" - "go.uber.org/zap" ) // Server that provides QEMU metadata. type Server struct { - log *logger.Logger - virt virConnect + log *slog.Logger + dhcpLeaseGetter LeaseGetter network string initSecretHashVal []byte } // New creates a new Server. -func New(log *logger.Logger, network, initSecretHash string, conn virConnect) *Server { +func New(log *slog.Logger, network, initSecretHash string, getter LeaseGetter) *Server { return &Server{ log: log, - virt: conn, + dhcpLeaseGetter: getter, network: network, initSecretHashVal: []byte(initSecretHash), } @@ -44,7 +42,6 @@ func (s *Server) ListenAndServe(port string) error { mux := http.NewServeMux() mux.Handle("/self", http.HandlerFunc(s.listSelf)) mux.Handle("/peers", http.HandlerFunc(s.listPeers)) - mux.Handle("/log", http.HandlerFunc(s.postLog)) mux.Handle("/endpoint", http.HandlerFunc(s.getEndpoint)) mux.Handle("/initsecrethash", http.HandlerFunc(s.initSecretHash)) @@ -57,25 +54,25 @@ func (s *Server) ListenAndServe(port string) error { return err } - s.log.Infof("Starting QEMU metadata API on %s", lis.Addr()) + s.log.Info(fmt.Sprintf("Starting QEMU metadata API on %s", lis.Addr())) return server.Serve(lis) } // listSelf returns peer information about the instance issuing the request. func (s *Server) listSelf(w http.ResponseWriter, r *http.Request) { - log := s.log.With(zap.String("peer", r.RemoteAddr)) - log.Infof("Serving GET request for /self") + log := s.log.With(slog.String("peer", r.RemoteAddr)) + log.Info("Serving GET request for /self") remoteIP, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - log.With(zap.Error(err)).Errorf("Failed to parse remote address") + log.With(slog.Any("error", err)).Error("Failed to parse remote address") http.Error(w, fmt.Sprintf("Failed to parse remote address: %s\n", err), http.StatusInternalServerError) return } peers, err := s.listAll() if err != nil { - log.With(zap.Error(err)).Errorf("Failed to list peer metadata") + log.With(slog.Any("error", err)).Error("Failed to list peer metadata") http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -87,23 +84,23 @@ func (s *Server) listSelf(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - log.Infof("Request successful") + log.Info("Request successful") return } } - log.Errorf("Failed to find peer in active leases") + log.Error("Failed to find peer in active leases") http.Error(w, "No matching peer found", http.StatusNotFound) } // listPeers returns a list of all active peers. func (s *Server) listPeers(w http.ResponseWriter, r *http.Request) { - log := s.log.With(zap.String("peer", r.RemoteAddr)) - log.Infof("Serving GET request for /peers") + log := s.log.With(slog.String("peer", r.RemoteAddr)) + log.Info("Serving GET request for /peers") peers, err := s.listAll() if err != nil { - log.With(zap.Error(err)).Errorf("Failed to list peer metadata") + log.With(slog.Any("error", err)).Error("Failed to list peer metadata") http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -113,46 +110,38 @@ func (s *Server) listPeers(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - log.Infof("Request successful") + log.Info("Request successful") } // initSecretHash returns the hash of the init secret. func (s *Server) initSecretHash(w http.ResponseWriter, r *http.Request) { - log := s.log.With(zap.String("initSecretHash", r.RemoteAddr)) + log := s.log.With(slog.String("initSecretHash", r.RemoteAddr)) if r.Method != http.MethodGet { - log.With(zap.String("method", r.Method)).Errorf("Invalid method for /initSecretHash") + log.With(slog.String("method", r.Method)).Error("Invalid method for /initSecretHash") http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - log.Infof("Serving GET request for /initsecrethash") + log.Info("Serving GET request for /initsecrethash") w.Header().Set("Content-Type", "text/plain") _, err := w.Write(s.initSecretHashVal) if err != nil { - log.With(zap.Error(err)).Errorf("Failed to write init secret hash") + log.With(slog.Any("error", err)).Error("Failed to write init secret hash") http.Error(w, err.Error(), http.StatusInternalServerError) return } - log.Infof("Request successful") + log.Info("Request successful") } // getEndpoint returns the IP address of the first control-plane instance. // This allows us to fake a load balancer for QEMU instances. func (s *Server) getEndpoint(w http.ResponseWriter, r *http.Request) { - log := s.log.With(zap.String("peer", r.RemoteAddr)) - log.Infof("Serving GET request for /endpoint") + log := s.log.With(slog.String("peer", r.RemoteAddr)) + log.Info("Serving GET request for /endpoint") - net, err := s.virt.LookupNetworkByName(s.network) + leases, err := s.dhcpLeaseGetter.GetDHCPLeases() if err != nil { - log.With(zap.Error(err)).Errorf("Failed to lookup network") - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer net.Free() - - leases, err := net.GetDHCPLeases() - if err != nil { - log.With(zap.Error(err)).Errorf("Failed to get DHCP leases") + log.With(slog.Any("error", err)).Error("Failed to get DHCP leases") http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -164,51 +153,18 @@ func (s *Server) getEndpoint(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - log.Infof("Request successful") + log.Info("Request successful") return } } - log.Errorf("Failed to find control-plane peer in active leases") + log.Error("Failed to find control-plane peer in active leases") http.Error(w, "No matching peer found", http.StatusNotFound) } -// postLog writes implements cloud-logging for QEMU instances. -func (s *Server) postLog(w http.ResponseWriter, r *http.Request) { - log := s.log.With(zap.String("peer", r.RemoteAddr)) - if r.Method != http.MethodPost { - log.With(zap.String("method", r.Method)).Errorf("Invalid method for /log") - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - log.Infof("Serving POST request for /log") - - if r.Body == nil { - log.Errorf("Request body is empty") - http.Error(w, "Request body is empty", http.StatusBadRequest) - return - } - - msg, err := io.ReadAll(r.Body) - if err != nil { - log.With(zap.Error(err)).Errorf("Failed to read request body") - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - log.With(zap.String("message", string(msg))).Infof("Cloud-logging entry") -} - // listAll returns a list of all active peers. func (s *Server) listAll() ([]metadata.InstanceMetadata, error) { - net, err := s.virt.LookupNetworkByName(s.network) - if err != nil { - return nil, err - } - defer net.Free() - - leases, err := net.GetDHCPLeases() + leases, err := s.dhcpLeaseGetter.GetDHCPLeases() if err != nil { return nil, err } @@ -231,6 +187,7 @@ func (s *Server) listAll() ([]metadata.InstanceMetadata, error) { return peers, nil } -type virConnect interface { - LookupNetworkByName(name string) (*virtwrapper.Network, error) +// LeaseGetter is an interface for getting DHCP leases. +type LeaseGetter interface { + GetDHCPLeases() ([]dhcp.NetworkDHCPLease, error) } diff --git a/hack/qemu-metadata-api/server/server_cgo_test.go b/hack/qemu-metadata-api/server/server_cgo_test.go deleted file mode 100644 index 59c569535..000000000 --- a/hack/qemu-metadata-api/server/server_cgo_test.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build cgo - -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package server - -import ( - "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/virtwrapper" - "libvirt.org/go/libvirt" -) - -type stubNetwork struct { - leases []libvirt.NetworkDHCPLease - getLeaseErr error -} - -func newStubNetwork(leases []virtwrapper.NetworkDHCPLease, getLeaseErr error) stubNetwork { - libvirtLeases := make([]libvirt.NetworkDHCPLease, len(leases)) - for i, l := range leases { - libvirtLeases[i] = libvirt.NetworkDHCPLease{ - IPaddr: l.IPaddr, - Hostname: l.Hostname, - } - } - return stubNetwork{ - leases: libvirtLeases, - getLeaseErr: getLeaseErr, - } -} - -func (n stubNetwork) GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error) { - return n.leases, n.getLeaseErr -} - -func (n stubNetwork) Free() error { - return nil -} diff --git a/hack/qemu-metadata-api/server/server_cross_test.go b/hack/qemu-metadata-api/server/server_cross_test.go deleted file mode 100644 index 3f4488b26..000000000 --- a/hack/qemu-metadata-api/server/server_cross_test.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build !cgo - -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package server - -import "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/virtwrapper" - -type stubNetwork struct { - leases []virtwrapper.NetworkDHCPLease - getLeaseErr error -} - -func newStubNetwork(leases []virtwrapper.NetworkDHCPLease, getLeaseErr error) stubNetwork { - return stubNetwork{ - leases: leases, - getLeaseErr: getLeaseErr, - } -} - -func (n stubNetwork) GetDHCPLeases() ([]virtwrapper.NetworkDHCPLease, error) { - return n.leases, n.getLeaseErr -} - -func (n stubNetwork) Free() error { - return nil -} diff --git a/hack/qemu-metadata-api/server/server_test.go b/hack/qemu-metadata-api/server/server_test.go index d9c78c2a5..ada18aaa3 100644 --- a/hack/qemu-metadata-api/server/server_test.go +++ b/hack/qemu-metadata-api/server/server_test.go @@ -1,22 +1,19 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package server import ( - "context" "encoding/json" - "errors" "io" "net/http" "net/http/httptest" - "strings" "testing" - "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/virtwrapper" + "github.com/edgelesssys/constellation/v2/hack/qemu-metadata-api/dhcp" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/stretchr/testify/assert" @@ -24,15 +21,13 @@ import ( ) func TestListAll(t *testing.T) { - someErr := errors.New("error") - testCases := map[string]struct { - wantErr bool - connect *stubConnect + wantErr bool + stubLeaseGetter *stubLeaseGetter }{ "success": { - connect: &stubConnect{ - network: newStubNetwork([]virtwrapper.NetworkDHCPLease{ + stubLeaseGetter: &stubLeaseGetter{ + leases: []dhcp.NetworkDHCPLease{ { IPaddr: "192.0.100.1", Hostname: "control-plane-0", @@ -45,20 +40,12 @@ func TestListAll(t *testing.T) { IPaddr: "192.0.200.1", Hostname: "worker-0", }, - }, nil), + }, }, }, - "LookupNetworkByName error": { - connect: &stubConnect{ - getNetworkErr: someErr, - }, - wantErr: true, - }, "GetDHCPLeases error": { - connect: &stubConnect{ - network: stubNetwork{ - getLeaseErr: someErr, - }, + stubLeaseGetter: &stubLeaseGetter{ + getErr: assert.AnError, }, wantErr: true, }, @@ -68,7 +55,7 @@ func TestListAll(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - server := New(logger.NewTest(t), "test", "initSecretHash", tc.connect) + server := New(logger.NewTest(t), "test", "initSecretHash", tc.stubLeaseGetter) res, err := server.listAll() @@ -77,58 +64,56 @@ func TestListAll(t *testing.T) { return } assert.NoError(err) - assert.Len(tc.connect.network.leases, len(res)) + assert.Len(tc.stubLeaseGetter.leases, len(res)) }) } } func TestListSelf(t *testing.T) { - someErr := errors.New("error") - testCases := map[string]struct { - remoteAddr string - connect *stubConnect - wantErr bool + remoteAddr string + stubLeaseGetter *stubLeaseGetter + wantErr bool }{ "success": { remoteAddr: "192.0.100.1:1234", - connect: &stubConnect{ - network: newStubNetwork([]virtwrapper.NetworkDHCPLease{ + stubLeaseGetter: &stubLeaseGetter{ + leases: []dhcp.NetworkDHCPLease{ { IPaddr: "192.0.100.1", Hostname: "control-plane-0", }, - }, nil), + }, }, }, "listAll error": { remoteAddr: "192.0.100.1:1234", - connect: &stubConnect{ - getNetworkErr: someErr, + stubLeaseGetter: &stubLeaseGetter{ + getErr: assert.AnError, }, wantErr: true, }, "remoteAddr error": { remoteAddr: "", - connect: &stubConnect{ - network: newStubNetwork([]virtwrapper.NetworkDHCPLease{ + stubLeaseGetter: &stubLeaseGetter{ + leases: []dhcp.NetworkDHCPLease{ { IPaddr: "192.0.100.1", Hostname: "control-plane-0", }, - }, nil), + }, }, wantErr: true, }, "peer not found": { remoteAddr: "192.0.200.1:1234", - connect: &stubConnect{ - network: newStubNetwork([]virtwrapper.NetworkDHCPLease{ + stubLeaseGetter: &stubLeaseGetter{ + leases: []dhcp.NetworkDHCPLease{ { IPaddr: "192.0.100.1", Hostname: "control-plane-0", }, - }, nil), + }, }, wantErr: true, }, @@ -139,9 +124,9 @@ func TestListSelf(t *testing.T) { assert := assert.New(t) require := require.New(t) - server := New(logger.NewTest(t), "test", "initSecretHash", tc.connect) + server := New(logger.NewTest(t), "test", "initSecretHash", tc.stubLeaseGetter) - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://192.0.0.1/self", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://192.0.0.1/self", nil) require.NoError(err) req.RemoteAddr = tc.remoteAddr @@ -158,22 +143,22 @@ func TestListSelf(t *testing.T) { var metadata metadata.InstanceMetadata require.NoError(json.Unmarshal(metadataRaw, &metadata)) - assert.Equal(tc.connect.network.leases[0].Hostname, metadata.Name) - assert.Equal(tc.connect.network.leases[0].IPaddr, metadata.VPCIP) + assert.Equal(tc.stubLeaseGetter.leases[0].Hostname, metadata.Name) + assert.Equal(tc.stubLeaseGetter.leases[0].IPaddr, metadata.VPCIP) }) } } func TestListPeers(t *testing.T) { testCases := map[string]struct { - remoteAddr string - connect *stubConnect - wantErr bool + remoteAddr string + stubNetworkGetter *stubLeaseGetter + wantErr bool }{ "success": { remoteAddr: "192.0.100.1:1234", - connect: &stubConnect{ - network: newStubNetwork([]virtwrapper.NetworkDHCPLease{ + stubNetworkGetter: &stubLeaseGetter{ + leases: []dhcp.NetworkDHCPLease{ { IPaddr: "192.0.100.1", Hostname: "control-plane-0", @@ -182,13 +167,13 @@ func TestListPeers(t *testing.T) { IPaddr: "192.0.200.1", Hostname: "worker-0", }, - }, nil), + }, }, }, "listAll error": { remoteAddr: "192.0.100.1:1234", - connect: &stubConnect{ - getNetworkErr: errors.New("error"), + stubNetworkGetter: &stubLeaseGetter{ + getErr: assert.AnError, }, wantErr: true, }, @@ -199,9 +184,9 @@ func TestListPeers(t *testing.T) { assert := assert.New(t) require := require.New(t) - server := New(logger.NewTest(t), "test", "initSecretHash", tc.connect) + server := New(logger.NewTest(t), "test", "initSecretHash", tc.stubNetworkGetter) - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://192.0.0.1/peers", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://192.0.0.1/peers", nil) require.NoError(err) req.RemoteAddr = tc.remoteAddr @@ -218,71 +203,23 @@ func TestListPeers(t *testing.T) { var metadata []metadata.InstanceMetadata require.NoError(json.Unmarshal(metadataRaw, &metadata)) - assert.Len(metadata, len(tc.connect.network.leases)) - }) - } -} - -func TestPostLog(t *testing.T) { - testCases := map[string]struct { - remoteAddr string - message io.Reader - method string - wantErr bool - }{ - "success": { - remoteAddr: "192.0.100.1:1234", - method: http.MethodPost, - message: strings.NewReader("test message"), - }, - "no body": { - remoteAddr: "192.0.100.1:1234", - method: http.MethodPost, - message: nil, - wantErr: true, - }, - "incorrect method": { - remoteAddr: "192.0.100.1:1234", - method: http.MethodGet, - message: strings.NewReader("test message"), - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - server := New(logger.NewTest(t), "test", "initSecretHash", &stubConnect{}) - - req, err := http.NewRequestWithContext(context.Background(), tc.method, "http://192.0.0.1/logs", tc.message) - require.NoError(err) - req.RemoteAddr = tc.remoteAddr - - w := httptest.NewRecorder() - server.postLog(w, req) - - if tc.wantErr { - assert.NotEqual(http.StatusOK, w.Code) - } else { - assert.Equal(http.StatusOK, w.Code) - } + assert.Len(metadata, len(tc.stubNetworkGetter.leases)) }) } } func TestInitSecretHash(t *testing.T) { - defaultConnect := &stubConnect{ - network: newStubNetwork([]virtwrapper.NetworkDHCPLease{ + defaultConnect := &stubLeaseGetter{ + leases: []dhcp.NetworkDHCPLease{ { IPaddr: "192.0.100.1", Hostname: "control-plane-0", }, - }, nil), + }, } + testCases := map[string]struct { - connect *stubConnect + connect *stubLeaseGetter method string wantHash string wantErr bool @@ -305,7 +242,7 @@ func TestInitSecretHash(t *testing.T) { server := New(logger.NewTest(t), "test", tc.wantHash, defaultConnect) - req, err := http.NewRequestWithContext(context.Background(), tc.method, "http://192.0.0.1/initsecrethash", nil) + req, err := http.NewRequestWithContext(t.Context(), tc.method, "http://192.0.0.1/initsecrethash", nil) require.NoError(err) w := httptest.NewRecorder() @@ -322,11 +259,11 @@ func TestInitSecretHash(t *testing.T) { } } -type stubConnect struct { - network stubNetwork - getNetworkErr error +type stubLeaseGetter struct { + leases []dhcp.NetworkDHCPLease + getErr error } -func (c stubConnect) LookupNetworkByName(_ string) (*virtwrapper.Network, error) { - return &virtwrapper.Network{Net: c.network}, c.getNetworkErr +func (c stubLeaseGetter) GetDHCPLeases() ([]dhcp.NetworkDHCPLease, error) { + return c.leases, c.getErr } diff --git a/hack/qemu-metadata-api/virtwrapper/virtwrapper_cgo.go b/hack/qemu-metadata-api/virtwrapper/virtwrapper_cgo.go deleted file mode 100644 index cda0bed96..000000000 --- a/hack/qemu-metadata-api/virtwrapper/virtwrapper_cgo.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build cgo - -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package virtwrapper - -import "libvirt.org/go/libvirt" - -// Connect wraps a libvirt connection. -type Connect struct { - Conn *libvirt.Connect -} - -// LookupNetworkByName looks up a network by name. -func (c *Connect) LookupNetworkByName(name string) (*Network, error) { - net, err := c.Conn.LookupNetworkByName(name) - if err != nil { - return nil, err - } - return &Network{Net: net}, nil -} - -// Network wraps a libvirt network. -type Network struct { - Net virNetwork -} - -// GetDHCPLeases returns the underlying DHCP leases. -func (n *Network) GetDHCPLeases() ([]NetworkDHCPLease, error) { - leases, err := n.Net.GetDHCPLeases() - if err != nil { - return nil, err - } - ret := make([]NetworkDHCPLease, len(leases)) - for i, l := range leases { - ret[i] = NetworkDHCPLease{ - IPaddr: l.IPaddr, - Hostname: l.Hostname, - } - } - return ret, nil -} - -// Free the network resource. -func (n *Network) Free() { - _ = n.Net.Free() -} - -type virNetwork interface { - GetDHCPLeases() ([]libvirt.NetworkDHCPLease, error) - Free() error -} diff --git a/hack/qemu-metadata-api/virtwrapper/virtwrapper_cross.go b/hack/qemu-metadata-api/virtwrapper/virtwrapper_cross.go deleted file mode 100644 index 2faa95961..000000000 --- a/hack/qemu-metadata-api/virtwrapper/virtwrapper_cross.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build !cgo - -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package virtwrapper - -import "errors" - -// Connect wraps a libvirt connection. -type Connect struct{} - -// LookupNetworkByName looks up a network by name. -// This function errors if CGO is disabled. -func (c *Connect) LookupNetworkByName(_ string) (*Network, error) { - return nil, errors.New("using virtwrapper requires building with CGO") -} - -// Network wraps a libvirt network. -type Network struct { - Net Net -} - -// GetDHCPLeases returns the underlying DHCP leases. -// This function errors if CGO is disabled. -func (n *Network) GetDHCPLeases() ([]NetworkDHCPLease, error) { - return n.Net.GetDHCPLeases() -} - -// Free the network resource. -// This function does nothing if CGO is disabled. -func (n *Network) Free() {} - -// Net is a libvirt Network. -type Net interface { - GetDHCPLeases() ([]NetworkDHCPLease, error) -} diff --git a/hack/tools/go.mod b/hack/tools/go.mod index f4d6754a0..0655ebf97 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -1,42 +1,54 @@ module github.com/edgelesssys/constellation/v2/hack/tools -go 1.21 +go 1.24.4 require ( github.com/google/go-licenses v1.6.0 - github.com/katexochen/sh/v3 v3.7.0 - golang.org/x/tools v0.14.0 - golang.org/x/vuln v1.0.1 + github.com/google/keep-sorted v0.6.1 + github.com/katexochen/sh/v3 v3.11.0 + golang.org/x/tools v0.35.0 + golang.org/x/vuln v1.1.4 ) require ( - github.com/emirpasic/gods v1.12.0 // indirect - github.com/go-logr/logr v1.2.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Workiva/go-datastructures v1.0.53 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/licenseclassifier v0.0.0-20210722185704-3043a050f148 // indirect github.com/google/renameio/v2 v2.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/otiai10/copy v1.6.0 // indirect - github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect - github.com/sergi/go-diff v1.2.0 // indirect - github.com/spf13/cobra v1.6.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/zerolog v1.31.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/src-d/gcfg v1.4.0 // indirect - github.com/xanzy/ssh-agent v0.2.1 // indirect - go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.16.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect + golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect gopkg.in/src-d/go-git.v4 v4.13.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - mvdan.cc/editorconfig v0.2.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + mvdan.cc/editorconfig v0.3.0 // indirect ) diff --git a/hack/tools/go.sum b/hack/tools/go.sum index a54164bc2..d45292e9b 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -59,7 +59,12 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Workiva/go-datastructures v1.0.53 h1:J6Y/52yX10Xc5JjXmGtWoSSxs3mZnGSaq37xZZh7Yig= +github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= @@ -83,14 +88,18 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -102,16 +111,18 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -163,12 +174,14 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-licenses v1.6.0 h1:MM+VCXf0slYkpWO0mECvdYDVCxZXIQNal5wqUIXEZ/A= github.com/google/go-licenses v1.6.0/go.mod h1:Z8jgz2isEhdenOqd/00pq7I4y4k1xVVQJv415otjclo= github.com/google/go-replayers/httpreplay v1.1.1 h1:H91sIMlt1NZzN7R+/ASswyouLJfW0WLW7fhyUFvDEkY= github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks= +github.com/google/keep-sorted v0.6.1 h1:LNEdDKYxoXOrn4ZXC+FdUfJCVbUjhb2QPIBs5XISXCI= +github.com/google/keep-sorted v0.6.1/go.mod h1:JYy9vljs7P8b3QdPOQkywA+4u36FUHwsNITZIpJyPkE= github.com/google/licenseclassifier v0.0.0-20210722185704-3043a050f148 h1:TJsAqW6zLRMDTyGmc9TPosfn9OyVlHs8Hrn3pY6ONSY= github.com/google/licenseclassifier v0.0.0-20210722185704-3043a050f148/go.mod h1:rq9F0RSpNKlrefnf6ZYMHKUnEJBCNzf6AcCXMYBeYvE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -214,19 +227,22 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/katexochen/sh/v3 v3.7.0 h1:jrU9BWBgp9o2NcetUVm3dNpQ2SK1zG6aF6WF0wtPajc= -github.com/katexochen/sh/v3 v3.7.0/go.mod h1:DSfEtJYp0xGV3Ex3oTePemLXE4F6rIj9hJtu2uYoDh0= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/katexochen/sh/v3 v3.11.0 h1:L71eTHDOVv7CKiCJVDm28EmIdIT/s8dXzKgddW9Uxok= +github.com/katexochen/sh/v3 v3.11.0/go.mod h1:Z6jje5FhaLunbZtZlnYi6qcaqMO+rdh8m0IJ2jWTbew= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -234,9 +250,14 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ= github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E= @@ -246,8 +267,7 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E= github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -256,15 +276,21 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97 h1:3RPlVWzZ/PDqmVuf/FKHARG5EMid/tl7cv54Sw/QRVY= -github.com/rogpeppe/go-internal v1.10.1-0.20230524175051-ec119421bb97/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= @@ -272,16 +298,23 @@ github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jW github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= +github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -294,8 +327,9 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -305,9 +339,10 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -345,8 +380,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -382,6 +417,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -392,8 +428,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -430,8 +466,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -443,6 +479,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -464,6 +501,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -493,17 +531,23 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b h1:DU+gwOBXU+6bO0sEyO7o/NeMlxZxCZEvI7v+J4a1zRQ= +golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b/go.mod h1:4ZwOYna0/zsOKwuR5X/m0QFOJpSZvAxFfkQT+Erd9D4= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -515,8 +559,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -562,6 +606,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -574,10 +619,14 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/vuln v1.0.1 h1:KUas02EjQK5LTuIx1OylBQdKKZ9jeugs+HiqO5HormU= -golang.org/x/vuln v1.0.1/go.mod h1:bb2hMwln/tqxg32BNY4CcxHWtHXuYa3SbIBmtsyjxtM= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools/go/expect v0.1.0-deprecated h1:jY2C5HGYR5lqex3gEniOQL0r7Dq5+VGVgY1nudX5lXY= +golang.org/x/tools/go/expect v0.1.0-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I= +golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -767,8 +816,9 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= @@ -781,6 +831,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -791,10 +842,11 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -mvdan.cc/editorconfig v0.2.0 h1:XL+7ys6ls/RKrkUNFQvEwIvNHh+JKx8Mj1pUV5wQxQE= -mvdan.cc/editorconfig v0.2.0/go.mod h1:lvnnD3BNdBYkhq+B4uBuFFKatfp02eB6HixDvEz91C0= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +mvdan.cc/editorconfig v0.3.0 h1:D1D2wLYEYGpawWT5SpM5pRivgEgXjtEXwC9MWhEY0gQ= +mvdan.cc/editorconfig v0.3.0/go.mod h1:NcJHuDtNOTEJ6251indKiWuzK6+VcrMuLzGMLKBFupQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/hack/tools/tools.go b/hack/tools/tools.go index 17a6b18e7..a7e28843b 100644 --- a/hack/tools/tools.go +++ b/hack/tools/tools.go @@ -3,13 +3,16 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ +// The tools module is used to keep tool dependencies separate from the main dependencies of the repo +// For more details see: https://github.com/golang/go/issues/25922#issuecomment-1038394599 package main import ( _ "github.com/google/go-licenses" + _ "github.com/google/keep-sorted" _ "github.com/katexochen/sh/v3/cmd/shfmt" _ "golang.org/x/tools/cmd/stringer" _ "golang.org/x/vuln/cmd/govulncheck" diff --git a/hack/versioninfogen/main.go b/hack/versioninfogen/main.go index 21183c131..c43e0488b 100644 --- a/hack/versioninfogen/main.go +++ b/hack/versioninfogen/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main diff --git a/image/BUILD.bazel b/image/BUILD.bazel index 681c6f385..772801b0c 100644 --- a/image/BUILD.bazel +++ b/image/BUILD.bazel @@ -1,16 +1,12 @@ load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file") load("@rules_pkg//:pkg.bzl", "pkg_tar") -load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") - -filegroup( - name = "sysroot_tree", - srcs = glob(["sysroot-tree/**"]), -) +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", "strip_prefix") pkg_files( name = "sysroot", - srcs = [":sysroot_tree"], - strip_prefix = strip_prefix.from_pkg() + "sysroot-tree", + srcs = glob(["sysroot-tree/**"]), + attributes = pkg_attributes(mode = "0555"), + strip_prefix = strip_prefix.from_pkg("sysroot-tree"), visibility = ["//visibility:public"], ) diff --git a/image/README.md b/image/README.md index 9d9de9952..c0d8c1975 100644 --- a/image/README.md +++ b/image/README.md @@ -4,6 +4,9 @@ Ensure you have Nix installed. This is a requirement for the following steps. Consult the [developer docs](/dev-docs/workflows/build-develop-deploy.md) for more info. At the very least, `nix` should be in your PATH. +Building the image also requires `newuidmap` and `newgidmap` to be present in the PATH. On Debian and Ubuntu, these can be sourced through +the `uidmap` package. + ## Build You can build any image using Bazel. @@ -16,113 +19,28 @@ bazel query //image/system/... You can either build a group of images (all images for a cloud provider, a stream, ...) or a single image by selecting a target. ```sh -bazel build //image/system:openstack_qemu-vtpm_debug +bazel build //image/system:azure_azure-sev-snp_stable ``` The location of the destination folder can be queried like this: ```sh -bazel cquery --output=files //image/system:openstack_qemu-vtpm_debug +bazel cquery --output=files //image/system:azure_azure-sev-snp_stable ``` -## Upload to CSP +## Build and Upload -Warning! Never set `--version` to a value that is already used for a release image. - -
-AWS - -- Install `aws` cli (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)) -- Login to AWS (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html)) -- Choose secure boot PKI public keys (one of `pki_dev`, `pki_test`, `pki_prod`) - - `pki_dev` can be used for local image builds - - `pki_test` is used by the CI for non-release images - - `pki_prod` is used for release images +Similarly, you can also build and upload images to the respective CSP within a single step with the `upload_*` targets. ```sh -# Warning! Never set `--version` to a value that is already used for a release image. -# Instead, use a `ref` that corresponds to your branch name. -bazel run //image/upload -- image aws --verbose --raw-image path/to/constellation.raw --attestation-variant "" --version ref/foo/stream/nightly/v2.7.0-pre-asdf +bazel run //image/system:upload_aws_aws-sev-snp_console -- --ref deps-image-fedora-40 --upload-measurements ``` -
+The `--ref` should be the branch you're building images on. It should **not contain slashes**. Slashes should be replaced with dashes to +not break the filesystem structure of the image storages. -
-GCP - -- Install `gcloud` and `gsutil` (see [here](https://cloud.google.com/sdk/docs/install)) -- Login to GCP (see [here](https://cloud.google.com/sdk/docs/authorizing)) -- Choose secure boot PKI public keys (one of `pki_dev`, `pki_test`, `pki_prod`) - - `pki_dev` can be used for local image builds - - `pki_test` is used by the CI for non-release images - - `pki_prod` is used for release images - -```sh -export GCP_RAW_IMAGE_PATH=$(realpath path/to/constellation.raw) -export GCP_IMAGE_PATH=path/to/image.tar.gz -upload/pack.sh gcp ${GCP_RAW_IMAGE_PATH} ${GCP_IMAGE_PATH} -# Warning! Never set `--version` to a value that is already used for a release image. -# Instead, use a `ref` that corresponds to your branch name. -bazel run //image/upload -- image gcp --verbose --raw-image "${GCP_IMAGE_PATH}" --attestation-variant "sev-es" --version ref/foo/stream/nightly/v2.7.0-pre-asdf -``` - -
- -
-Azure - -Note: - -> For testing purposes, it is a lot simpler to disable Secure Boot for the uploaded image! -> Disabling Secure Boot allows you to skip the VMGS creation steps above. - -- Install `az` and `azcopy` (see [here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli)) -- Login to Azure (see [here](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli)) -- Optional (if Secure Boot should be enabled) [Prepare virtual machine guest state (VMGS) with customized NVRAM or use existing VMGS blob](#azure-secure-boot) - -```sh -export AZURE_RAW_IMAGE_PATH=path/to/constellation.raw -export AZURE_IMAGE_PATH=path/to/image.vhd -upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}" -# Warning! Never set `--version` to a value that is already used for a release image. -# Instead, use a `ref` that corresponds to your branch name. -bazel run //image/upload -- image azure --verbose --raw-image "${AZURE_IMAGE_PATH}" --attestation-variant "cvm" --version ref/foo/stream/nightly/v2.7.0-pre-asdf -``` - -
- -
-OpenStack - -Note: - -> OpenStack is not one a global cloud provider, but rather a software that can be installed on-premises. -> This means we do not upload the image to a cloud provider, but to our CDN. - -- Install `aws` cli (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)) -- Login to AWS (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html)) - -```sh -# Warning! Never set `--version` to a value that is already used for a release image. -# Instead, use a `ref` that corresponds to your branch name. -bazel run //image/upload -- image openstack --verbose --raw-image path/to/constellation.raw --attestation-variant "sev" --version ref/foo/stream/nightly/v2.7.0-pre-asdf -``` - -
- -
-QEMU - -- Install `aws` cli (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)) -- Login to AWS (see [here](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-quickstart.html)) - -```sh -# Warning! Never set `--version` to a value that is already used for a release image. -# Instead, use a `ref` that corresponds to your branch name. -bazel run //image/upload -- image qemu --verbose --raw-image path/to/constellation.raw --attestation-variant "default" --version ref/foo/stream/nightly/v2.7.0-pre-asdf -``` - -
+Optionally, the `--upload-measurements` option can be used to specify that measurements for the image should be uploaded, and `--fake-sign` specifies +that a debugging signing key should be used to sign the measurements, which is done for debug images. ## Kernel @@ -131,3 +49,26 @@ We track the latest longterm release, use sources directly from [kernel.org](htt srpm spec file. After building a Kernel rpm, we upload it to our CDN and use it in our image builds. + +## Upgrading to a new Fedora release + +- Search for the old Fedora releasever in the `image/` directory and replace every occurence (outside of lockfiles) with the new releasever +- Search for Fedora container images in Dockerfiles and upgrade the releasever +- Regenerate the package lockfile: `bazel run //image/mirror:update_packages` +- Build test images locally: + - `bazel query //image/system:all` (pick an image name from the output) + - `bazel build //image/system:IMAGE_NAME_HERE` (replace with an actual image name) +- Let CI build new images and run e2e tests +- Upgrade kernel spec under [edgelesssys/constellation-kernel](https://github.com/edgelesssys/constellation-kernel) to use new releasever + +## Adding new packages to the image + +- Find the package (i.e. it's _package name_) on [Koji](https://koji.fedoraproject.org/koji/) +- Add the package to the corresponding section in `./base/mkosi.conf` + - If the package is required to be present in the initrd, add it to `./initrd/mkosi.conf` +- Add the package to `./mirror/packages.txt` +- Update the package mirror: + ```sh + bazel run //image/mirror:update_packages + ``` +- Build new images (e.g. via CI) and run e2e tests diff --git a/image/base/BUILD.bazel b/image/base/BUILD.bazel index 1d02317b2..9e613d4fd 100644 --- a/image/base/BUILD.bazel +++ b/image/base/BUILD.bazel @@ -1,87 +1,100 @@ -load("@aspect_bazel_lib//lib:copy_file.bzl", "copy_file") load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", "strip_prefix") load("//bazel/mkosi:mkosi_image.bzl", "mkosi_image") copy_to_directory( - name = "rpms", + name = "rpms_lts", srcs = [ - "@kernel//file", - "@kernel_core//file", - "@kernel_modules//file", - "@kernel_modules_core//file", + "@kernel_core_lts//file", + "@kernel_lts//file", + "@kernel_modules_core_lts//file", + "@kernel_modules_lts//file", ], - include_external_repositories = ["kernel*"], + include_external_repositories = ["kernel*lts"], + replace_prefixes = {"file": ""}, +) + +copy_to_directory( + name = "rpms_mainline", + srcs = [ + "@kernel_core_mainline//file", + "@kernel_mainline//file", + "@kernel_modules_core_mainline//file", + "@kernel_modules_mainline//file", + ], + include_external_repositories = ["kernel*mainline"], replace_prefixes = {"file": ""}, ) [ - copy_file( - name = name, - src = "@" + name + "//file", - out = name + ".rpm", - allow_symlink = True, + mkosi_image( + name = "base_" + kernel_variant, + srcs = [ + "mkosi.finalize", + "mkosi.postinst", + "mkosi.prepare", + ], + outs = [ + kernel_variant, + kernel_variant + ".tar", + kernel_variant + "-.rpm.lock", + kernel_variant + "-packagemanifest", + kernel_variant + "-rpmdb.sqlite", + kernel_variant + "-rpmdb.sqlite-shm", + kernel_variant + "-rpmdb.sqlite-wal", + ], + extra_trees = [ + ":skeleton", + "//image:sysroot_tar", + "//image:cryptsetup_closure", + ], + local_mirror = ["@mkosi_rpms//:repo"], + mkosi_conf = "mkosi.conf", + output = kernel_variant, + package_directories = [ + ":rpms_" + kernel_variant, + ], + tags = [ + "manual", + "no-cache", + ], + visibility = ["//visibility:public"], ) - for name in [ - "kernel", - "kernel_core", - "kernel_modules", - "kernel_modules_core", + for kernel_variant in [ + "lts", + "mainline", ] ] -mkosi_image( - name = "base", - srcs = [ - "mkosi.postinst", - "mkosi.prepare", - ] + glob([ - "mkosi.skeleton/**", - ]), - outs = [ - "image", - "image.tar", - "image-.rpm.lock", - "image-packagemanifest", - "image-rpmdb.sqlite", - "image-rpmdb.sqlite-shm", - "image-rpmdb.sqlite-wal", - ], - extra_trees = [ - "//image:sysroot_tar", - "//image:cryptsetup_closure", - ], - local_mirror = ["@mkosi_rpms//:repo"], - mkosi_conf = "mkosi.conf", - package_files = [ - ":kernel", - ":kernel_core", - ":kernel_modules", - ":kernel_modules_core", - ], - tags = [ - "manual", - "no-cache", - ], - visibility = ["//visibility:public"], -) - pkg_tar( name = "rpmdb", srcs = [ - "image-.rpm.lock", - "image-packagemanifest", - "image-rpmdb.sqlite", - "image-rpmdb.sqlite-shm", - "image-rpmdb.sqlite-wal", + "lts-.rpm.lock", + "lts-packagemanifest", + "lts-rpmdb.sqlite", + "lts-rpmdb.sqlite-shm", + "lts-rpmdb.sqlite-wal", ], remap_paths = { - "/image-.rpm.lock": "/var/lib/rpm/.rpm.lock", - "/image-packagemanifest": "/usr/share/constellation/packagemanifest", - "/image-rpmdb.sqlite": "/var/lib/rpm/rpmdb.sqlite", - "/image-rpmdb.sqlite-shm": "/var/lib/rpm/rpmdb.sqlite-shm", - "/image-rpmdb.sqlite-wal": "/var/lib/rpm/image-rpmdb.sqlite-wal", + "/lts-.rpm.lock": "/var/lib/rpm/.rpm.lock", + "/lts-packagemanifest": "/usr/share/constellation/packagemanifest", + "/lts-rpmdb.sqlite": "/var/lib/rpm/rpmdb.sqlite", + "/lts-rpmdb.sqlite-shm": "/var/lib/rpm/rpmdb.sqlite-shm", + "/lts-rpmdb.sqlite-wal": "/var/lib/rpm/image-rpmdb.sqlite-wal", }, tags = ["manual"], visibility = ["//visibility:public"], ) + +pkg_files( + name = "skeleton_files", + srcs = glob(["mkosi.skeleton/**"]), + attributes = pkg_attributes(mode = "0555"), + strip_prefix = strip_prefix.from_pkg("mkosi.skeleton"), +) + +pkg_tar( + name = "skeleton", + srcs = [":skeleton_files"], +) diff --git a/image/base/mkosi.conf b/image/base/mkosi.conf index 4ee2f73fc..9201a05ff 100644 --- a/image/base/mkosi.conf +++ b/image/base/mkosi.conf @@ -1,6 +1,7 @@ [Distribution] Distribution=fedora -Release=38 +Release=40 +RepositoryKeyFetch=yes [Output] Format=tar @@ -15,6 +16,9 @@ Packages=systemd dbus udev util-linux + kernel + kernel-core + kernel-modules # nvme / disk / udev tools Packages=nvme-cli @@ -37,6 +41,7 @@ Packages=containerd # Network Packages=iproute dbus + openssh-server systemd-networkd systemd-resolved diff --git a/image/base/mkosi.finalize b/image/base/mkosi.finalize new file mode 100755 index 000000000..561db202f --- /dev/null +++ b/image/base/mkosi.finalize @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# For some reason yet unknown, SourceDateEpoch is not applied correctly to the +# users added by systemd-sysusers. This has only been observed in our mkosi +# flake so far, not in an upstream mkosi configuration. +# TODO(burgerdev): wait for a couple of Nix package upgrades and try again? + +# Strategy: unset the "last password change" date without leaving a trace in +# /etc/shadow-. +tmp=$(mktemp) +cp -a "${BUILDROOT}/etc/shadow-" "${tmp}" +mkosi-chroot chage -d "" etcd +cp -a "${tmp}" "${BUILDROOT}/etc/shadow-" diff --git a/image/base/mkosi.postinst b/image/base/mkosi.postinst index 62b680654..728e5ad17 100755 --- a/image/base/mkosi.postinst +++ b/image/base/mkosi.postinst @@ -15,3 +15,19 @@ cp "${BUILDROOT}/usr/share/constellation/packagemanifest" "${OUTPUTDIR}/" # copy rpmdb to outputs cp "${BUILDROOT}"/var/lib/rpm/{rpmdb.sqlite-wal,rpmdb.sqlite-shm,rpmdb.sqlite,.rpm.lock} "${OUTPUTDIR}/" + +# FIXME(msanft): +# Hack to satisfy Bazel's [output expectations](./BUILD.bazel). +# 2 Bazel packages can't share the same output paths, as it seems, and the +# files being copied around here aren't large, so copying them around doesn't +# hurt. +cp "${OUTPUTDIR}/packagemanifest" "${OUTPUTDIR}/lts-packagemanifest" +cp "${OUTPUTDIR}/.rpm.lock" "${OUTPUTDIR}/lts-.rpm.lock" +cp "${OUTPUTDIR}/rpmdb.sqlite" "${OUTPUTDIR}/lts-rpmdb.sqlite" +cp "${OUTPUTDIR}/rpmdb.sqlite-shm" "${OUTPUTDIR}/lts-rpmdb.sqlite-shm" +cp "${OUTPUTDIR}/rpmdb.sqlite-wal" "${OUTPUTDIR}/lts-rpmdb.sqlite-wal" +cp "${OUTPUTDIR}/packagemanifest" "${OUTPUTDIR}/mainline-packagemanifest" +cp "${OUTPUTDIR}/.rpm.lock" "${OUTPUTDIR}/mainline-.rpm.lock" +cp "${OUTPUTDIR}/rpmdb.sqlite" "${OUTPUTDIR}/mainline-rpmdb.sqlite" +cp "${OUTPUTDIR}/rpmdb.sqlite-shm" "${OUTPUTDIR}/mainline-rpmdb.sqlite-shm" +cp "${OUTPUTDIR}/rpmdb.sqlite-wal" "${OUTPUTDIR}/mainline-rpmdb.sqlite-wal" diff --git a/image/base/mkosi.skeleton/usr/etc/containers/containers.conf b/image/base/mkosi.skeleton/etc/containers/containers.conf similarity index 100% rename from image/base/mkosi.skeleton/usr/etc/containers/containers.conf rename to image/base/mkosi.skeleton/etc/containers/containers.conf diff --git a/image/base/mkosi.skeleton/usr/etc/containers/registries.conf b/image/base/mkosi.skeleton/etc/containers/registries.conf similarity index 100% rename from image/base/mkosi.skeleton/usr/etc/containers/registries.conf rename to image/base/mkosi.skeleton/etc/containers/registries.conf diff --git a/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-bootstrapper.service b/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-bootstrapper.service index 76ef974ce..30ca0acfe 100644 --- a/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-bootstrapper.service +++ b/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-bootstrapper.service @@ -1,7 +1,8 @@ [Unit] Description=Constellation Bootstrapper Wants=network-online.target -After=network-online.target configure-constel-csp.service +Requires=sshd-keygen.target +After=network-online.target configure-constel-csp.service sshd-keygen.target After=export_constellation_debug.service [Service] @@ -10,7 +11,7 @@ RemainAfterExit=yes Restart=on-failure EnvironmentFile=/run/constellation.env Environment=PATH=/run/state/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin -ExecStart=/usr/bin/bootstrapper $CONSTELLATION_DEBUG_FLAGS +ExecStart=/usr/bin/bootstrapper [Install] WantedBy=multi-user.target diff --git a/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-upgrade-agent.service b/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-upgrade-agent.service index c3fefdcc5..ffa204085 100644 --- a/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-upgrade-agent.service +++ b/image/base/mkosi.skeleton/usr/lib/systemd/system/constellation-upgrade-agent.service @@ -8,7 +8,7 @@ RemainAfterExit=yes Restart=on-failure EnvironmentFile=/run/constellation.env Environment=PATH=/run/state/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin -ExecStart=/usr/bin/upgrade-agent $CONSTELLATION_DEBUG_FLAGS +ExecStart=/usr/bin/upgrade-agent [Install] WantedBy=multi-user.target diff --git a/image/base/mkosi.skeleton/usr/lib/systemd/system/containerd.service.d/local.conf b/image/base/mkosi.skeleton/usr/lib/systemd/system/containerd.service.d/local.conf index e1c02c704..30143c9ae 100644 --- a/image/base/mkosi.skeleton/usr/lib/systemd/system/containerd.service.d/local.conf +++ b/image/base/mkosi.skeleton/usr/lib/systemd/system/containerd.service.d/local.conf @@ -1,3 +1,11 @@ [Service] ExecStart= ExecStart=/usr/bin/containerd --config /usr/etc/containerd/config.toml + +# Until https://github.com/containerd/containerd/pull/8924 lands in our +# containerd version, we need to decrease the default ulimit that +# is set in the upstream containerd service of infinity which newer +# systemd versions resolve to 2**30 which is way too large and +# results in various inspecific errors such as excessive resource (e.g., memory) +# usage. +LimitNOFILE=524288 diff --git a/image/base/mkosi.skeleton/usr/lib/systemd/system/export_constellation_debug.service b/image/base/mkosi.skeleton/usr/lib/systemd/system/export_constellation_debug.service index 6858dab9b..9b0fccabe 100644 --- a/image/base/mkosi.skeleton/usr/lib/systemd/system/export_constellation_debug.service +++ b/image/base/mkosi.skeleton/usr/lib/systemd/system/export_constellation_debug.service @@ -3,7 +3,7 @@ Description=Export Constellation Debug Level to Environment [Service] Type=oneshot -ExecStart=/bin/bash -c "tr ' ' '\n' < /proc/cmdline | grep -q 'constellation.debug' && echo CONSTELLATION_DEBUG_FLAGS=--debug >> /run/constellation.env" +ExecStart=/bin/bash -c "tr ' ' '\n' < /proc/cmdline | grep -q 'constel.debug' && echo CONSTELLATION_DEBUG_FLAGS=--debug >> /run/constellation.env" RemainAfterExit=yes [Install] diff --git a/image/initrd/BUILD.bazel b/image/initrd/BUILD.bazel index 682c9d70e..a959018ec 100644 --- a/image/initrd/BUILD.bazel +++ b/image/initrd/BUILD.bazel @@ -1,18 +1,18 @@ +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", "strip_prefix") load("//bazel/mkosi:mkosi_image.bzl", "mkosi_image") mkosi_image( name = "initrd", srcs = [ "mkosi.postinst", - ] + glob([ - "mkosi.skeleton/**", - "reposdir/**", - ]), + ], outs = [ "image", "image.cpio.zst", ], extra_trees = [ + ":skeleton", "//image:sysroot_tar", "//image:cryptsetup_closure", "//disk-mapper/cmd:disk-mapper-package.tar", @@ -25,3 +25,15 @@ mkosi_image( ], visibility = ["//visibility:public"], ) + +pkg_files( + name = "skeleton_files", + srcs = glob(["mkosi.skeleton/**"]), + attributes = pkg_attributes(mode = "0555"), + strip_prefix = strip_prefix.from_pkg("mkosi.skeleton"), +) + +pkg_tar( + name = "skeleton", + srcs = [":skeleton_files"], +) diff --git a/image/initrd/mkosi.conf b/image/initrd/mkosi.conf index 9c32e11ad..bceb53c8a 100644 --- a/image/initrd/mkosi.conf +++ b/image/initrd/mkosi.conf @@ -1,6 +1,7 @@ [Distribution] Distribution=fedora -Release=38 +Release=40 +RepositoryKeyFetch=yes [Output] Format=cpio diff --git a/image/measured-boot/cmd/main.go b/image/measured-boot/cmd/main.go index 9037709fd..1cf708ba3 100644 --- a/image/measured-boot/cmd/main.go +++ b/image/measured-boot/cmd/main.go @@ -1,13 +1,14 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main import ( "bytes" + "crypto" "crypto/sha256" "encoding/json" "fmt" @@ -93,7 +94,7 @@ func measurePE(fs afero.Fs, peFile string) ([]byte, error) { } defer f.Close() - return measure.Authentihash(f, sha256.New()) + return measure.Authentihash(f, crypto.SHA256) } func precalculatePCR4(simulator *measure.Simulator, fs afero.Fs, ukiFile string) error { @@ -111,7 +112,7 @@ func precalculatePCR4(simulator *measure.Simulator, fs afero.Fs, ukiFile string) if err != nil { return fmt.Errorf("uki does not contain linux kernel image: %v", err) } - linuxMeasurement, err := measure.Authentihash(linuxSectionReader, sha256.New()) + linuxMeasurement, err := measure.Authentihash(linuxSectionReader, crypto.SHA256) if err != nil { return fmt.Errorf("failed to measure linux kernel image: %v", err) } diff --git a/image/measured-boot/extract/extract.go b/image/measured-boot/extract/extract.go index d96c302e0..cd544a7ae 100644 --- a/image/measured-boot/extract/extract.go +++ b/image/measured-boot/extract/extract.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package extract @@ -90,9 +90,8 @@ var ukiSections = []string{ ".initrd", ".splash", ".dtb", - // uanme and sbat will be added in systemd-stub >= 254 - // ".uname", - // ".sbat", + ".uname", + ".sbat", ".pcrsig", ".pcrkey", } diff --git a/image/measured-boot/extract/extract_test.go b/image/measured-boot/extract/extract_test.go index 4e8e6379c..ec1b161aa 100644 --- a/image/measured-boot/extract/extract_test.go +++ b/image/measured-boot/extract/extract_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package extract @@ -19,7 +19,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestPeSectionReader(t *testing.T) { @@ -114,6 +114,26 @@ func TestPeFileSectionDigests(t *testing.T) { }, Measure: true, MeasureOrder: 5, }, + { + Name: ".uname", Size: 0x22, + Digest: [32]uint8{ + 0x32, 0xd5, 0x9d, 0x99, 0x0e, 0x9c, 0x1f, 0x7d, + 0xa5, 0x54, 0xcb, 0x88, 0x8e, 0x32, 0x38, 0xac, + 0x61, 0x93, 0xe5, 0xe7, 0x23, 0x0f, 0x99, 0xb1, + 0x97, 0x13, 0x8d, 0xd7, 0x23, 0xc0, 0xeb, 0xb6, + }, + Measure: true, MeasureOrder: 6, + }, + { + Name: ".sbat", Size: 0x10, + Digest: [32]uint8{ + 0x66, 0x30, 0xfb, 0x7d, 0x5b, 0xaf, 0x9d, 0x6c, + 0xd5, 0x1c, 0x9a, 0xc9, 0x54, 0x10, 0xe6, 0x8a, + 0xa3, 0xfe, 0xdb, 0x4a, 0xdd, 0xd4, 0x2b, 0x34, + 0x0e, 0x47, 0x11, 0xe2, 0x3c, 0xcc, 0xd4, 0xb2, + }, + Measure: true, MeasureOrder: 7, + }, { Name: ".pcrkey", Size: 0x12, @@ -123,7 +143,7 @@ func TestPeFileSectionDigests(t *testing.T) { 0x69, 0xd0, 0x86, 0xa6, 0xd6, 0x7d, 0x5f, 0xee, 0x88, 0xdb, 0x21, 0x90, 0xc4, 0xa7, 0x07, 0x26, }, - Measure: true, MeasureOrder: 7, + Measure: true, MeasureOrder: 9, }, { Name: ".data", @@ -181,16 +201,6 @@ func TestPeFileSectionDigests(t *testing.T) { }, Measure: false, MeasureOrder: -1, }, - { - Name: ".sbat", Size: 0x10, - Digest: [32]uint8{ - 0x66, 0x30, 0xfb, 0x7d, 0x5b, 0xaf, 0x9d, 0x6c, - 0xd5, 0x1c, 0x9a, 0xc9, 0x54, 0x10, 0xe6, 0x8a, - 0xa3, 0xfe, 0xdb, 0x4a, 0xdd, 0xd4, 0x2b, 0x34, - 0x0e, 0x47, 0x11, 0xe2, 0x3c, 0xcc, 0xd4, 0xb2, - }, - Measure: false, MeasureOrder: -1, - }, { Name: ".sdmagic", Size: 0x2d, Digest: [32]uint8{ @@ -211,16 +221,6 @@ func TestPeFileSectionDigests(t *testing.T) { }, Measure: false, MeasureOrder: -1, }, - { - Name: ".uname", Size: 0x22, - Digest: [32]uint8{ - 0x32, 0xd5, 0x9d, 0x99, 0x0e, 0x9c, 0x1f, 0x7d, - 0xa5, 0x54, 0xcb, 0x88, 0x8e, 0x32, 0x38, 0xac, - 0x61, 0x93, 0xe5, 0xe7, 0x23, 0x0f, 0x99, 0xb1, - 0x97, 0x13, 0x8d, 0xd7, 0x23, 0xc0, 0xeb, 0xb6, - }, - Measure: false, MeasureOrder: -1, - }, { Name: ".pcrsig", Size: 0x216, Digest: [32]uint8{ @@ -229,7 +229,7 @@ func TestPeFileSectionDigests(t *testing.T) { 0xb8, 0x13, 0xb5, 0x31, 0xb0, 0x56, 0x3e, 0x91, 0x20, 0x55, 0x6c, 0xf7, 0x25, 0x01, 0xa3, 0x26, }, - Measure: false, MeasureOrder: 6, + Measure: false, MeasureOrder: 8, }, }, sectionDigests) diff --git a/image/measured-boot/fixtures/fixtures.go b/image/measured-boot/fixtures/fixtures.go index 0e9372594..6e4662037 100644 --- a/image/measured-boot/fixtures/fixtures.go +++ b/image/measured-boot/fixtures/fixtures.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package fixtures diff --git a/image/measured-boot/measure/BUILD.bazel b/image/measured-boot/measure/BUILD.bazel index 7d3358c06..0fc8089ac 100644 --- a/image/measured-boot/measure/BUILD.bazel +++ b/image/measured-boot/measure/BUILD.bazel @@ -14,7 +14,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//image/measured-boot/pesection", - "@com_github_foxboron_go_uefi//efi/pecoff", + "@com_github_foxboron_go_uefi//authenticode", "@org_golang_x_text//encoding/unicode", ], ) diff --git a/image/measured-boot/measure/authentihash.go b/image/measured-boot/measure/authentihash.go index 718577d8c..a6facd887 100644 --- a/image/measured-boot/measure/authentihash.go +++ b/image/measured-boot/measure/authentihash.go @@ -1,31 +1,42 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure import ( "bytes" + "crypto" "fmt" - "hash" "io" - "github.com/foxboron/go-uefi/efi/pecoff" + "github.com/foxboron/go-uefi/authenticode" ) // Authentihash returns the PE/COFF hash / Authentihash of a file. -func Authentihash(r io.Reader, h hash.Hash) ([]byte, error) { +func Authentihash(r io.Reader, h crypto.Hash) ([]byte, error) { + readerAt, err := getReaderAt(r) + if err != nil { + return nil, fmt.Errorf("failed to get readerAt: %v", err) + } + + bin, err := authenticode.Parse(readerAt) + if err != nil { + return nil, fmt.Errorf("failed to parse pe file: %v", err) + } + return bin.Hash(h), nil +} + +func getReaderAt(r io.Reader) (io.ReaderAt, error) { + if ra, ok := r.(io.ReaderAt); ok { + return ra, nil + } + buf := new(bytes.Buffer) if _, err := buf.ReadFrom(r); err != nil { return nil, fmt.Errorf("failed to read pe file: %v", err) } - - signingCtx := pecoff.PECOFFChecksum(buf.Bytes()) - pecoff.PaddSigCtx(signingCtx) - - h.Write(signingCtx.SigData.Bytes()) - - return h.Sum(nil), nil + return bytes.NewReader(buf.Bytes()), nil } diff --git a/image/measured-boot/measure/authentihash_test.go b/image/measured-boot/measure/authentihash_test.go index e451460af..9032e6cbf 100644 --- a/image/measured-boot/measure/authentihash_test.go +++ b/image/measured-boot/measure/authentihash_test.go @@ -1,14 +1,14 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure import ( "bytes" - "crypto/sha256" + "crypto" "testing" "github.com/edgelesssys/constellation/v2/image/measured-boot/fixtures" @@ -19,7 +19,7 @@ func TestPeSectionReader(t *testing.T) { assert := assert.New(t) peReader := bytes.NewReader(fixtures.UKI()) - digest, err := Authentihash(peReader, sha256.New()) + digest, err := Authentihash(peReader, crypto.SHA256) assert.NoError(err) assert.Equal( []byte{ diff --git a/image/measured-boot/measure/measure_test.go b/image/measured-boot/measure/measure_test.go index 899b2c15c..7c37fc864 100644 --- a/image/measured-boot/measure/measure_test.go +++ b/image/measured-boot/measure/measure_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure @@ -13,5 +13,5 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } diff --git a/image/measured-boot/measure/pcr.go b/image/measured-boot/measure/pcr.go index f90ddeef2..d44ea3271 100644 --- a/image/measured-boot/measure/pcr.go +++ b/image/measured-boot/measure/pcr.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr04.go b/image/measured-boot/measure/pcr04.go index 6a343bee9..5117860ee 100644 --- a/image/measured-boot/measure/pcr04.go +++ b/image/measured-boot/measure/pcr04.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr04_test.go b/image/measured-boot/measure/pcr04_test.go index 215fb5d8e..5b9e7f568 100644 --- a/image/measured-boot/measure/pcr04_test.go +++ b/image/measured-boot/measure/pcr04_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr09.go b/image/measured-boot/measure/pcr09.go index 8013f3c39..5a6f279e1 100644 --- a/image/measured-boot/measure/pcr09.go +++ b/image/measured-boot/measure/pcr09.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr09_test.go b/image/measured-boot/measure/pcr09_test.go index 2462605e9..759ad9e33 100644 --- a/image/measured-boot/measure/pcr09_test.go +++ b/image/measured-boot/measure/pcr09_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr11.go b/image/measured-boot/measure/pcr11.go index 1fbc37887..6f5d8d789 100644 --- a/image/measured-boot/measure/pcr11.go +++ b/image/measured-boot/measure/pcr11.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr11_test.go b/image/measured-boot/measure/pcr11_test.go index ca94c16de..5b73aec2e 100644 --- a/image/measured-boot/measure/pcr11_test.go +++ b/image/measured-boot/measure/pcr11_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/measure/pcr_test.go b/image/measured-boot/measure/pcr_test.go index d05f47b69..37487219f 100644 --- a/image/measured-boot/measure/pcr_test.go +++ b/image/measured-boot/measure/pcr_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measure diff --git a/image/measured-boot/pesection/pesection.go b/image/measured-boot/pesection/pesection.go index 59b849e39..557851cf8 100644 --- a/image/measured-boot/pesection/pesection.go +++ b/image/measured-boot/pesection/pesection.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package pesection diff --git a/image/mirror/BUILD.bazel b/image/mirror/BUILD.bazel index 56b425add..014001e9d 100644 --- a/image/mirror/BUILD.bazel +++ b/image/mirror/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") + sh_binary( name = "update_packages", srcs = ["update_packages.sh"], diff --git a/image/mirror/SHA256SUMS b/image/mirror/SHA256SUMS index fc07f6d90..796826902 100644 --- a/image/mirror/SHA256SUMS +++ b/image/mirror/SHA256SUMS @@ -1,354 +1,360 @@ -49750ebd8f565bdb9c94250faa419863d533e826661f19bdbeab40d14461a80c WALinuxAgent-udev-2.9.0.4-1.fc38.noarch.rpm -215a4863c02da27447378371075fc48e57b99e10e98b28347f9bbb06e37ae139 aardvark-dns-1.9.0-1.fc38.x86_64.rpm -53572033e3dbec252fc928cd04eb93a49509f4a62a31582853248bead759d0a8 alternatives-1.25-1.fc38.x86_64.rpm -9d228124e13e4fcffb65a3c8cab1a8c6356912e3adca77ee9830d789b5e897ab audit-libs-3.1.2-5.fc38.i686.rpm -8e52e3cc2d7d1f8d55f5490a07d53a0b8f61a4d4eb0fd4a3f072383455e75d91 audit-libs-3.1.2-5.fc38.x86_64.rpm -17f200a45179c59193ab7c72a4641b502ab5c524f0e5a0d59fd95aa6f15bffc8 authselect-1.4.3-1.fc38.x86_64.rpm -d2f324c915eb5e14542a55636c8e49c1cd75b17db1a8c7b11693a989629a250b authselect-libs-1.4.3-1.fc38.x86_64.rpm -718d95c40b41c2f0ecc8dc2290ebb91b529ba3be7accbad9c30c88e9ce408349 basesystem-11-15.fc38.noarch.rpm -6a63fec13b7df170eefb2f5d6f1db01af8ce15019a9fceaa04783799c4943170 bash-5.2.21-1.fc38.x86_64.rpm -0551362b7c7efc44188147b482179f477240cb8146129d68e9f8a584af0cee68 bzip2-libs-1.0.8-13.fc38.i686.rpm -95273426afa05a81e6cf77f941e1156f6a0a3305a7768a02c04a4164280cf876 bzip2-libs-1.0.8-13.fc38.x86_64.rpm -43df24cf3974e8f8b5472a0389d519282ee55c7a720460351d0c75c4866ccf19 ca-certificates-2023.2.60_v7.0.306-1.0.fc38.noarch.rpm -152f433af3f709a3f5060505f5b3b009e3da8ea455d6d1dab5c3ed2173f9e016 catatonit-0.1.7-14.fc38.x86_64.rpm -eec292696b1d3db55380f3fc6f4a04eac76cc8a09fa8c795783e9af3c6385627 conmon-2.1.8-2.fc38.x86_64.rpm -17dfa6a009e8e80119911c2fbde44b43425f2bfe9abf677f68657f0d96603d2a conntrack-tools-1.4.7-1.fc38.x86_64.rpm -b17dfa3be93f927e066839e646c8bc613e57e6d59bd0cf0cf5c9cf67ade7c363 container-selinux-2.226.0-1.fc38.noarch.rpm -d9fb9be5604e68ea59a87c14c09f89cdbaddba7ec5761ecded9211ad976db910 containerd-1.6.19-1.fc38.x86_64.rpm -db4691e52e7d8b9614445428ef8f43a91b0d7069ffd07000c606e7d562df9804 containernetworking-plugins-1.3.0-2.fc38.x86_64.rpm -cd9583500892b5f50f51167f5c4b16855e15808a79e5137e0c6674e19f305930 containers-common-1-89.fc38.noarch.rpm -66f5951f5ae9c8ec9cddf1ff87b0c8aea29298c763712bfec0ed507f57b6d3e6 containers-common-extra-1-89.fc38.noarch.rpm -1100feca307bb7159a00f336fa5303fd52ded761fc25c77fdc5f095e2add29b3 coreutils-single-9.1-12.fc38.x86_64.rpm -75ecf8c60bea53432b973d9391e3bdb1b6cdc1f1cad0a7b56aabfb8b70252832 cpio-2.13-14.fc38.x86_64.rpm -65ed3fa87ea138a51a5280848aac565e996c3217fa9cd9eafa8a4a2c05f7b37d cracklib-2.9.11-1.fc38.i686.rpm -9e13cb10c94f5de3573a2cacd153050d6dad05fe13c50c8fa69406e8f881d3d9 cracklib-2.9.11-1.fc38.x86_64.rpm -c09679c55fdeef598c6cd9118e8a2141da369dcf27ed9e9ede8e42ffc44743f2 cracklib-dicts-2.9.11-1.fc38.x86_64.rpm -3c6985fe1a0388482131411b5a0811658411056ac7b96687225bf77143645ca4 criu-3.18-1.fc38.x86_64.rpm -979323d3e3e891195ebb48edbf1e741f14937ecd3e3639f8c86eabdce0c30b94 criu-libs-3.18-1.fc38.x86_64.rpm -6076cb7d1146b417d2836b465ec584fcdaf6e9bb40f55c0a6f2527c8ab2767b7 crun-1.12-1.fc38.x86_64.rpm -6809fe060dc93dfed723bd8faca0f5d65e4f6c62aebd2f9f39f679413b260539 crypto-policies-20230301-1.gita12f7b2.fc38.noarch.rpm -d1880b8f3e8a5fa943ba033c96dc964fac3e07c34cbd0bb8ac2b0fdcb57abcbc crypto-policies-scripts-20230301-1.gita12f7b2.fc38.noarch.rpm -418287cd51f9ae9e5b74a18112a062a49ee51d22656dea86d08daba311c93e44 cryptsetup-2.6.1-1.fc38.x86_64.rpm -364cfba8a49773110ea200a533037afd99307b80a28333fc5f900f528a66333e cryptsetup-libs-2.6.1-1.fc38.i686.rpm -070d86aca4548e9148901881a4ef64d98c5dfd4ea158e9208c650c7f4b225c47 cryptsetup-libs-2.6.1-1.fc38.x86_64.rpm -61e1f069ced21f62d486b0dacc14aed2a7fdbbcff84ad40ef9eedd6b450f4afb curl-8.0.1-6.fc38.x86_64.rpm -9400386fc0a427faa3d07c1d43d99f2e49bfcfda5c16d0df035329cd1e6c69d6 curl-minimal-8.0.1-6.fc38.x86_64.rpm -b570b4857289cf32ed57d0d84cb861677a649cef7c5cc2498a249d6593eb3614 cyrus-sasl-lib-2.1.28-9.fc38.x86_64.rpm -faa01592782cad3faec0c0a07a3fa17bd0a793dc906fcb7ffedb975f1d077be4 dbus-1.14.10-1.fc38.x86_64.rpm -6652f5d40acfaeda20555bdd63e964f123e8ab4a52f8c8890e749e06b6bae3e0 dbus-broker-33-1.fc38.x86_64.rpm -a97ebbbaf97348a0bcf967eb8c3c822fc30154cf9679f5a1cb21f7c507077bbd dbus-common-1.14.10-1.fc38.noarch.rpm -695aab11fda4b6a396c3ca141fb3ccc48af757a70f36ea3a1be2292e90c40d6f device-mapper-1.02.189-2.fc38.x86_64.rpm -4be737f08ab8e072cad67bcfa0063d87b55aa4ada132f5d442c2c1bc40205599 device-mapper-libs-1.02.189-2.fc38.i686.rpm -42e8600a8d7e7109de32df6cb6a619b9805f9335bf467a085009f6b82dda6e22 device-mapper-libs-1.02.189-2.fc38.x86_64.rpm -f20b884c9452d4cf839707d4ff38191d34dd2035c87f832c697f6f328fa1b36b diffutils-3.10-1.fc38.x86_64.rpm -9b82b697e3f3f7cb1c6c411a142b09fe71562e7ac34778b1f1c5e6b4d232a8cc dracut-059-5.fc38.x86_64.rpm -8bcdb9b5ce9b5e19c4f772f52c2c40712491d2a953496e415c380a21a792b050 duktape-2.7.0-2.fc38.x86_64.rpm -b25a042f52c8cdc1ddab56fc726d98789ff902d5264d9835f5b910db3e565213 e2fsprogs-1.46.5-4.fc38.x86_64.rpm -1ca049bff926a8ec9b6e0e69f23662934eab96c35d8eda1cf429a2a31f186045 e2fsprogs-libs-1.46.5-4.fc38.x86_64.rpm -3e0ec4d7b4b95d10f58c5269688e03da7abb4d73169c76761f4fc7e7f7797a47 ec2-utils-1.2-47.amzn2.noarch.rpm -5b5195bb6dacdd7c78522675ce83784e2befd1bdbadb14c609d9e1488754ca79 efitools-1.9.2-9.fc38.x86_64.rpm -fbeb7a7a95051df0dfeb7c4ab379b77ccfb0bf0174a39dae4a423b9108627771 efivar-libs-38-7.fc38.x86_64.rpm -4a554b06daad6d5afae8b49a2615f1d5330c9cb4f0a78287a41fa9d0946d4827 elfutils-debuginfod-client-0.190-2.fc38.i686.rpm -8e77dbab6df03188bad35c578a3054fa9a547b5049f34000f8a3cde722b24c75 elfutils-debuginfod-client-0.190-2.fc38.x86_64.rpm -05841b0b924df5959f3820e0fe3926c1aedb833d20e5c72828560cb23b81a585 elfutils-default-yama-scope-0.190-2.fc38.noarch.rpm -d03a7939acff945b2e9fe35ccfca1d908f75d9c99b1ed52e88c97214076a7ae8 elfutils-libelf-0.190-2.fc38.i686.rpm -b6e80be0b89f98498ae91cd10e22ac837118f55ed3e1d19c0bbc450dc2795d27 elfutils-libelf-0.190-2.fc38.x86_64.rpm -3b76bc2b1546cc433e327767796a84287e0439a89df1d77d05eb5182c087f9d5 elfutils-libs-0.190-2.fc38.i686.rpm -373913149a0086f4b2bb1d73cab7270ec5e8432aa451fa5aa46c7705aec4e105 elfutils-libs-0.190-2.fc38.x86_64.rpm -086ae7cbe09c1f206fcc833adb2f7d265c2c28a3a93fd3a86fb51dfbcc01e9e5 ethtool-6.6-1.fc38.x86_64.rpm -6c64fea958acfb77da5ee23ec1e8d0916c7809ce39987f219927e8f94e5f2755 expat-2.5.0-2.fc38.x86_64.rpm -7f7c78f598f7ff131bbe77913b9fc6b7b49d1fce30f2d8505b2d8a85458f519a fedora-gpg-keys-38-1.noarch.rpm -40f7d64e38ae31dcb3e7273c7e089705e816dc131ae87c176e2c1aad02d5b510 fedora-release-38-36.noarch.rpm -ac9ede79357b33f0d0c9087333b0dd3e3fd1cf5ccab5c36310b0ec446390e0c7 fedora-release-common-38-36.noarch.rpm -bacf386d747343cb10a2c3847c426d3e044b7de1742b4918e3b1b78cc1b54bc4 fedora-release-identity-basic-38-36.noarch.rpm -916b75b58e9a2afe5d53cb73fdabea4d0cd8b1eba9f1213754384d0ccd531e57 fedora-repos-38-1.noarch.rpm -196b3612e50069c68d67ffaf3081d25923e2b4ca7e1bad8f124515b3fa472d4a file-5.44-3.fc38.x86_64.rpm -83bb8576a018ae028ff1434c32e1f3dfab66a781e1af9057862a2d7f3d725f58 file-libs-5.44-3.fc38.x86_64.rpm -b0fc6c55f5989aebf6e71279541206070b32b3b28b708a249bd3bdeaa6c088a4 filesystem-3.18-3.fc38.x86_64.rpm -79986f917ef1bae7ca2378b16515ba44c19160f5a5eae4f6b697eda160bc26c1 findutils-4.9.0-3.fc38.x86_64.rpm -d5ae6a7d99826a17d163d9846c2705442b5792a7ccacc5169e4986cdf4b6bae2 fuse-common-3.14.1-1.fc38.x86_64.rpm -56df47937646df892dad25c6b9ae63d111328febfe86eb93096b8b0a11700b60 fuse-libs-2.9.9-16.fc38.x86_64.rpm -1e9e8b6447c2650a306e8d107dbdcdaa4f81d4175012eea0c87846faecd64c70 fuse-overlayfs-1.12-1.fc38.x86_64.rpm -55ca555fe815bd360b08500889b652f50fe4c56dfafbed0cc459f2362641f1a0 fuse3-3.14.1-1.fc38.x86_64.rpm -f54340fec047cc359a6a164a1ce88d0d7ffcd8f7d6334b50dc5b3d234e3a19ac fuse3-libs-3.14.1-1.fc38.x86_64.rpm -e607df61803999da46a199d23d4acadb45b290f29b5b644e583c5526d8081178 gawk-5.1.1-5.fc38.x86_64.rpm -254c6789b4a98b96a53172e1b3fb866f71ea29a5b5fa7b39f072f33c27d897bc gawk-all-langpacks-5.1.1-5.fc38.x86_64.rpm -eb376264750aae673aa2a3218c291756023dea980640b30a3efe0f2199ff3889 gdbm-libs-1.23-3.fc38.x86_64.rpm -276c7af4b13262c0307cf2528c6d79c859ed348db68c0d2780869ea4b179dd02 gettext-envsubst-0.21.1-2.fc38.x86_64.rpm -4fb6fcf7eef64a48666ff9fe5a46344087979d9e8fd87be4d58a17cf9c3ef108 gettext-libs-0.21.1-2.fc38.x86_64.rpm -3837cbe450ceb59e1f9e7469aeb6ec98e08150773b83463725acfb2ebb77a98a gettext-runtime-0.21.1-2.fc38.x86_64.rpm -99d5c1b62e51a0cbba487122e28fb795ebfcacbba35cccd1260a29b55aff221f glib2-2.76.6-1.fc38.x86_64.rpm -8960e2a305292ef35784d954830aaee4efaefa52f2010e3bc1932dc9794868b7 glibc-2.37-16.fc38.i686.rpm -da8c74d2085230b590b9d45e046affdaa8be60b7c677eac18d91e5033c9ccd0c glibc-2.37-16.fc38.x86_64.rpm -b5829081347ac9d27fb805d88c2d5fb48f159327d8b4cc5d96aabce648db6bec glibc-common-2.37-16.fc38.x86_64.rpm -48aae24ddbe4e84e9f6e5770607c5611b0e9085b44b0f4e0a36c6e9a3eb37311 glibc-gconv-extra-2.37-16.fc38.i686.rpm -1d30f8c16ca4def36de67ef6992bcb29175a31964ae5077d50eac5b719b11fa3 glibc-gconv-extra-2.37-16.fc38.x86_64.rpm -a658f84f03aeb4ab9721cb3ab54750a4df3aa31c82b2f6ee8cab06a414038b6a glibc-minimal-langpack-2.37-16.fc38.x86_64.rpm -69e48c73d962e3798fbc84150dfd79258b32a4f9250ee801d8cdb9540edfc21a gmp-6.2.1-4.fc38.x86_64.rpm -940192d99005896a29de3c682259b3505652778d58ec74a003639f3ba3193a1a gnupg2-2.4.0-3.fc38.x86_64.rpm -cd41c94b8c668602f7fb5eae595e5d5c34bd1b91690b5cc06f4c8c199794dfa8 gnupg2-smime-2.4.0-3.fc38.x86_64.rpm -d592abfff7d8d2c7ec9a0882bfd0903a8dc7ca2bd132f97d8c06cea4509ee1f1 gnutls-3.8.1-1.fc38.x86_64.rpm -86475210e5d0994dbc614bdc2ad1bd85d37f1e6c2aeee08f41d803195691ee89 google-compute-engine-guest-configs-udev-20230929.00-1.fc38.noarch.rpm -e0481a0fd263907193fe9f3f080a17e89de1ef1d8a490078a6225062b4eec761 gpgme-1.17.1-5.fc38.x86_64.rpm -ad16ec814c4423d007d218a3f45d2e39d3dab00fc8c0d75eef176041594e3970 gpm-libs-1.20.7-42.fc38.x86_64.rpm -60ed241ec381a23d03fac733a72132dbdc4ba04c412add78bfc67f1b9f1b4daa grep-3.8-3.fc38.x86_64.rpm -550d58a0d2eba399a6648b5de2a50685bbecd2a8bf5a8317bd70e0c8fb1d3c84 grub2-common-2.06-108.fc38.noarch.rpm -b2f4d1c35345d29b998b523e315797d634ba16cf510e7d2a0640bc2188a8ba98 grub2-tools-2.06-108.fc38.x86_64.rpm -104c3297dc36fc7a22a4fe7c2b44941ad891a76c674b4360b87163daca244bf5 grub2-tools-minimal-2.06-108.fc38.x86_64.rpm -d48579aa97fba34dc3a4c4dd3fb10e8fd66186d970e37c2f7fc28bebbc2d31eb grubby-8.40-69.fc38.x86_64.rpm -90fb8fe8a8dbe9bad49dbecf3f6b9e2ed7e699a68489a4839a05de1c4678dfdc gvisor-tap-vsock-0.7.1-1.fc38.x86_64.rpm -fcfc9d204d917c9557bc6017879e601995e60d0ff10075c7b8ab88e57f14b2af gvisor-tap-vsock-gvforwarder-0.7.1-1.fc38.x86_64.rpm -166e842798813a72e4d075dcd9b6e814d7ad3fc8d1b5860281cffe7a68784b25 gzip-1.12-3.fc38.x86_64.rpm -95c1a55fd14757d33b9c8e2a85dbcda5601bc20c166f983dacbb410849fad4da iproute-6.4.0-1.fc38.x86_64.rpm -74e4a1b67ba1772df7bedbb2510a45feb110c58902c805cd105e9693c6875ddd iproute-tc-6.4.0-1.fc38.x86_64.rpm -86906f04a9f2e96289ae3dd71e3069af4fd42df7de87d7853c790f50f5baeeb6 iptables-legacy-1.8.9-4.fc38.1.x86_64.rpm -abcb71db54b690f8100f34f201fba60c21eae13e052f6b221eda7e1ee8bcc04c iptables-legacy-libs-1.8.9-4.fc38.1.x86_64.rpm -767c6eccb8dda9f9617d245233822f91c0ed478afc421c1a1ff09584b6484ade iptables-libs-1.8.9-4.fc38.1.x86_64.rpm -d5f329b2d0ec7ed50b047701f7c54b37f5ffabc27e57c2fb5462b32bc9b41583 iptables-nft-1.8.9-4.fc38.1.x86_64.rpm -f74d1e8ba95ff5df02477ba6430d09476e8a1af6e8476453d08d99b276924b25 jansson-2.13.1-6.fc38.x86_64.rpm -7fa521efbcc6a27d679a2ad6ab7d07a8a742d9de49458d4b4352f528c8c78e1d json-c-0.17-1.fc38.i686.rpm -d111f87bed4f918bbd720d509fd97072b345f3a5998910d0a0413104b6fe98c2 json-c-0.17-1.fc38.x86_64.rpm -8a3a4007d921c6a9b372e4573520866b445c18476f907b023ea19b436436ec33 kbd-2.5.1-5.fc38.x86_64.rpm -97fcab8f93c5213714e0a9d15e48440c567d2ac39e4ed12743198268082f0583 kbd-legacy-2.5.1-5.fc38.noarch.rpm -2ea5dfd9d8c8fe7f61022b41fd6f8a2653692acf30253f45e1dc67def1648f32 kbd-misc-2.5.1-5.fc38.noarch.rpm -578e492c3e144f3d0469420823d2d3733b2c0428e1ecc3b542f3d8b465567762 keyutils-libs-1.6.1-6.fc38.i686.rpm -b66376e78fe54024531d94036d10b22b5050d52a9e65682b36dc7fdf24405997 keyutils-libs-1.6.1-6.fc38.x86_64.rpm -7419671c64795b96be18231e2f5d3f95eca8e6a71771863ac035f961041c1d7c kmod-30-4.fc38.x86_64.rpm -b4021011438edc3e8a8ef6535f84e72829f6346716d079bd2cf032f72deff3f7 kmod-libs-30-4.fc38.i686.rpm -19f873b67f23362074c03d5825e709ad521278c02e44bdeb30eba6d3bb3a8e0f kmod-libs-30-4.fc38.x86_64.rpm -ac8a7628a2a4b1f742fc90719145f50cfad9ecf67e3309fcf623fb3d82c2a768 kpartx-0.9.4-2.fc38.x86_64.rpm -96c1846341e14589a7c66f4cf21bcc0788225b44ce652c5729299a701d31a2fa krb5-libs-1.21-3.fc38.i686.rpm -9ea8a9ddec5b16c157a9b3a6661957da3688969fa94c55ae9e0ba4fcb07539b9 krb5-libs-1.21-3.fc38.x86_64.rpm -5d959bcbd96ad757865da02c799babce8a7f7fd71d740b616215ea2d0d00a985 libacl-2.3.1-6.fc38.i686.rpm -9b093be8a99bfbae03c2f3dd5435fc9508003f7ef21e4280ff72fe814c1d794e libacl-2.3.1-6.fc38.x86_64.rpm -0d0890dba8274458d068b981164111f554b47632bf9c82d1ec41c14697f2b4af libarchive-3.6.1-4.fc38.x86_64.rpm -0ffba864821a315749a7ad1773eb43e506d26d4ef633cd0940188e76add8ebd4 libargon2-20190702-2.fc38.i686.rpm -dd044973c572e64f505f3d00482249b2b4d71369babb5395a22861fd55b21d79 libargon2-20190702-2.fc38.x86_64.rpm -03c7ea2565af3b61a6ff0d5d4e0e5c65747053fc43903b913a46337903c6fdb5 libassuan-2.5.6-1.fc38.x86_64.rpm -8b43d1f6b92bf174e95ec60e2d6f0e5b2ed76177515c4e13e2b202d9a594c139 libattr-2.5.1-6.fc38.i686.rpm -d78d7bc485f099bb08c9de55dd12ea6a984b948face1f947de6ec805663a96c5 libattr-2.5.1-6.fc38.x86_64.rpm -9e73a2b591ebf2915bfbe7f9479498a73c982a4c74e96cc555930022e3ef0aba libb2-0.98.1-8.fc38.x86_64.rpm -dca5cafabf192d1f5abe37fa06425877bf74bb6e8c5ce5cad577274b18169b94 libblkid-2.38.1-4.fc38.i686.rpm -21b5a1a024c2d1877d2b7271fd3f82424eb0bd6b95395ad3a3dae5776eec8714 libblkid-2.38.1-4.fc38.x86_64.rpm -8079443881e764cece2f8f6789b39ebbe43226cde61675bdfae5a5a18a439b5f libbpf-1.1.0-2.fc38.x86_64.rpm -faea0f6f20d52f95031352fb3f7a35d5fe2065b736694decee5728fa9a153907 libbsd-0.11.7-4.fc38.x86_64.rpm -34a1a7ecb75ee3646015f5f6a89baec0e9aecb00d3cc99f60c9195342763957f libcap-2.48-7.fc38.i686.rpm -0d75699c16fb36d308fec83bf3eac626a1dcbafb267025340b8019be8e780de8 libcap-2.48-7.fc38.x86_64.rpm -5257031cba9a8791a277994e026b0f4c7a1cf2878505f5e1ed463fa670b67f05 libcap-ng-0.8.3-8.fc38.i686.rpm -c0b6770708d273bcbf83d78f87353ca03629150e7a2f0efcadc24c41ca140500 libcap-ng-0.8.3-8.fc38.x86_64.rpm -5922028bb5642faf00d781f34bf105ef30f1988932b4b80f6bb54e9f6eed0fd6 libcbor-0.7.0-9.fc38.x86_64.rpm -ba35e2297e78af6c55bbb2a3e46a1d0819ced548088023d32f8f5dec2a9dfb88 libcom_err-1.46.5-4.fc38.i686.rpm -4ed3e7b6b0727b86ae9af17bd4839c06762a417a263a5d22eb7fcb39714bb480 libcom_err-1.46.5-4.fc38.x86_64.rpm -8835069e84135c300459175ad85bc78f278263a392cfdf692e13c6e99c9d2c49 libcurl-minimal-8.0.1-6.fc38.i686.rpm -891b9b131d7b467a55864150a059d70c5dcc0e27696ab19e326decd3c95ab6d7 libcurl-minimal-8.0.1-6.fc38.x86_64.rpm -d7030af9e9e0dd9afc5b9ee02839c94757d6a4f8ebd3b21e6d81ba6141a76d46 libdb-5.3.28-55.fc38.x86_64.rpm -377c739918e9636e36218c7074316b7817e7cba9cff862198c478fb1c028112c libeconf-0.5.2-1.fc38.i686.rpm -56768f18f856353a2344f47f065b5f6dff164a5188d466df4bc64d675f455e42 libeconf-0.5.2-1.fc38.x86_64.rpm -974a64a10a3021de8a440ff4810a720f738951abd5bb944110cb9355d4ae8fa8 libedit-3.1-45.20221030cvs.fc38.x86_64.rpm -e9741c40e94cf45bdc699b950c238646c2d56b3ee7984e748b94d8e6f87ba3cd libevent-2.1.12-8.fc38.x86_64.rpm -768346084add1264110b08521694b0b25506b9c8b4bdbc53dc492b05cf70d052 libfdisk-2.38.1-4.fc38.i686.rpm -2fb7ee2d94f7ee34cff49ab28659c07b075ed67ac147f817e19d8ee8e0adbc9c libfdisk-2.38.1-4.fc38.x86_64.rpm -cd08ba5d43459f6e3bbb3cc86caee43b74b38d551d69eae3e86861c2622da1cd libffi-3.4.4-2.fc38.i686.rpm -098e8ba05482205c70c3510907da71819faf5d40b588f865ad2a77d6eaf4af09 libffi-3.4.4-2.fc38.x86_64.rpm -0a30628b39d9c5ca81c4e073dfbf64d543284f17d4ae1325e23e3eda55f92fd9 libfido2-1.12.0-3.fc38.x86_64.rpm -10bef1a628b5691f7dc0c5e17922e98b914522c6f7c7cc02b496f63c79a57f18 libgcc-13.2.1-4.fc38.i686.rpm -18142b95b05275d198bb77cefbf0be8f2c29c60293e2e99b013f6f7e28595969 libgcc-13.2.1-4.fc38.x86_64.rpm -ef4b2686134e6be036755ee093aad43eb0ce4a4c84c93a2defb755cfeb398754 libgcrypt-1.10.2-1.fc38.x86_64.rpm -fc552c5c10bccff56930ef74422c1774dd6fd4900f61376f7602ae3f05de52d9 libgomp-13.2.1-4.fc38.x86_64.rpm -40b98bdd00b44fbf808df1d653ecf0f0ed8549643d52b68da2455c5a878f0021 libgpg-error-1.47-1.fc38.x86_64.rpm -9f2059c5d699f3dd2337f0872968123a06cf56b9f349d58bd64a5ef22a9815b4 libibverbs-44.0-3.fc38.x86_64.rpm -6890b8a81db38f0d03635c69d1b641e5bba302938f3c0c49d715b90651c5b06f libidn2-2.3.4-2.fc38.i686.rpm -d3416e2b6c7565d7a607225d86b556398827316ae7ce43280b82630f0a022bc0 libidn2-2.3.4-2.fc38.x86_64.rpm -75c0097330fa3c995e80b7791cbe7baf75d86f3523f67b3becaf37360fdb4b16 libkcapi-1.4.0-5.fc38.x86_64.rpm -e552fae193775507d8264f7a856fbdc51f7e594d7d8726f181312aeb9cf8b973 libkcapi-hmaccalc-1.4.0-5.fc38.x86_64.rpm -4c5af4c1d44e1720beb79fd4202d3754e2c47c4faeb4ff5bdbe2ee3d24c9a5e0 libksba-1.6.4-1.fc38.x86_64.rpm -96213462b3d77ef8e73d72510dd70210e9af4ae28f7466b88f2740743e6ca49d libmd-1.1.0-1.fc38.x86_64.rpm -f79da3b0edf002221250bce8352015009f8259793278ae59cb74d6c9e0c8395b libmetalink-0.1.3-30.fc38.x86_64.rpm -729b80bbf6ca427c21dd28d1f7029b736dc42f3f4f0c43d322ddfa168bc2ce9b libmnl-1.0.5-2.fc38.x86_64.rpm -34b0fbfe9493d0e0762bfe42137238f3eb9cee0832e1d816f5c9f0a48ac798e9 libmount-2.38.1-4.fc38.i686.rpm -14541cda2a4516ad6a17f46be6b7ad85ef5f6508d36f209f2ba7bd45bc1504e2 libmount-2.38.1-4.fc38.x86_64.rpm -db30396e0f1eb0ac81d353524bc2e022371a5e5a3bed363e101e461d8d248fca libnet-1.3-1.fc38.x86_64.rpm -cc6bca4b52d6aaca9b1cc1f4f02721301fba447a3d2f009a7b9e9c38da3eb10f libnetfilter_conntrack-1.0.9-1.fc38.x86_64.rpm -0054a179032b916330c202a4a5713d9403fedc0809ffc215e6345705f869633c libnetfilter_cthelper-1.0.0-23.fc38.x86_64.rpm -95a9876660af858339b38a2272c7307917f4cc15a6f4ffac4fbd4d6f6743473f libnetfilter_cttimeout-1.0.0-21.fc38.x86_64.rpm -e86f7341f80e9143a9cc1df8b7fcf864eccdea487fc34500b412678501b37146 libnetfilter_queue-1.0.5-4.fc38.x86_64.rpm -3c981697fe61f23ad41b615b4c3197d023ec70f713395fc3104b837c61b74294 libnfnetlink-1.0.1-23.fc38.x86_64.rpm -06f6aeaded732bcff2d7dd5c8b1c430bebc3834d0968f20e3c2918ae15502ace libnftnl-1.2.4-2.fc38.x86_64.rpm -13161773e6afa5439ff8ef19faaa3baec35930a07df21901b01726eeda4e2644 libnghttp2-1.52.0-2.fc38.i686.rpm -9daf3a24341811a5feded45fe3e111a1d01f6a91854c231ff56fa8dffb8d40f9 libnghttp2-1.52.0-2.fc38.x86_64.rpm -a9d80e55bd59e26338a7778de28caf9eb3874f8d90574c879bae1302beaa862b libnl3-3.7.0-3.fc38.x86_64.rpm -28697cf1b5cb4d62c3bd154fc24a23d91a84a5bda2f974fb64bdd04e91b6cec5 libnsl2-2.0.0-5.fc38.x86_64.rpm -bf1e07244e3c9aacfe96b2b7f21e7bb678d1c52042885d3f0518301de38dd759 libnvme-1.4-2.fc38.x86_64.rpm -f4d87eb23450cd3888af3d984ad623d229e5bea482188d25f996e61a939486bf libpcap-1.10.4-1.fc38.x86_64.rpm -e0bccc94a740acf317caa4fa1fae6b0d57442ef4be36341472b7db93d588ec13 libpsl-0.21.2-2.fc38.x86_64.rpm -4625cab157ff1760c2f5053a3f75cec21b366f80c3b1ff53bf0db1a033a28439 libpwquality-1.4.5-3.fc38.i686.rpm -aefb7d2d96af03f4d7ac5a132138d383faf858011b1740c48fcd152000f3c617 libpwquality-1.4.5-3.fc38.x86_64.rpm -e3ef79196bc8cb77c35bd20f265f5a551ec3e8482a82ad92af3e802f9a302ad7 libseccomp-2.5.3-4.fc38.i686.rpm -dec378b594b79258dd8b44836c5371f316bcf5e4596d53dd84badcb6d00090df libseccomp-2.5.3-4.fc38.x86_64.rpm -46ed6b8fee11c16bb8b58f698dfba9874a8f393c1e72eb7f9a7b6802ac68dd1a libsecret-0.20.5-3.fc38.x86_64.rpm -9c938bd9917a9f977ab0572e1cea573f2a886a0a0a48587463faa5ed1d6b22e0 libselinux-3.5-1.fc38.i686.rpm -790c6d821ff575ad51242ec6832ed61c8a3c4e0ece245c3dee3292d19acb23b7 libselinux-3.5-1.fc38.x86_64.rpm -78a15621e7e3dfb5a65b8b8aa482cf5b07f08bcef217ad29435e299d6c8aec74 libselinux-utils-3.5-1.fc38.x86_64.rpm -1b6b7ad33391919a3315e398d737a764121e2fc9581f75318a999e02bfc0c7c4 libsemanage-3.5-2.fc38.x86_64.rpm -14292c07496f6db6ef27913d6d144e01ce7017c57ef990edff3d04a443e5507d libsepol-3.5-1.fc38.i686.rpm -15ec70665f200a5423589539c3253677eb3c15d7d620fd9bdfe2d1e429735198 libsepol-3.5-1.fc38.x86_64.rpm -ac0a6bf295151973d2e34392a134e246560b19b7351ced244abc1ed81dfe5b8e libsigsegv-2.14-4.fc38.x86_64.rpm -b35a0d6b1ecb151982b6a9342d00e8da9663e8a6da6b21b7c559634f7f29fd2d libslirp-4.7.0-3.fc38.x86_64.rpm -b71b1633a2b514d27dd9332d733f61ddd8c5a501360f9e8bafee313b793d6ad0 libsmartcols-2.38.1-4.fc38.i686.rpm -dbf5c73c71c798533cbecfa54ba28c42878c455df8cb382087d8a758c3ffe290 libsmartcols-2.38.1-4.fc38.x86_64.rpm -17da6760ce632b1726de4291f287f2928a70becc1ed414bb2e8b359dd1b7d815 libsodium-1.0.18-11.fc38.x86_64.rpm -faccff819eecffcee9dad49bda930a007e78b905b775b4ac0103121d7a8100db libss-1.46.5-4.fc38.x86_64.rpm -c70b6fa3ddd6765e2b1c9a8eddb4c0446214c55afa85fed7e79dfc98d91d10ee libstdc++-13.2.1-4.fc38.x86_64.rpm -261adde7d452da52848797f1ed22a219e86a8cf516d2028b0b5f892c63aedf6c libtasn1-4.19.0-2.fc38.i686.rpm -8b49dd88579f1c37e05780202e81022c9400422b830d9bdd9087161683628b22 libtasn1-4.19.0-2.fc38.x86_64.rpm -355966acde2fb387628248c18ca8f6cfc928300cee3aee667e36208f932de815 libtirpc-1.3.4-0.fc38.x86_64.rpm -aa187ea45be32306620ad8ec6318d755075b2cad99fba7c01dc4763228a98190 libtool-ltdl-2.4.7-6.fc38.x86_64.rpm -c4012952872a08b9662963b13e29f89388ce6e695e68fa8c37eb6e62bad62441 libunistring-1.1-3.fc38.x86_64.rpm -882075e0d35161170ea243a7d318e51dd45c51b9c7b749ef7d8cbbab4ab75895 libunistring1.0-1.0-1.fc38.i686.rpm -cd0e8eb5d983a985f7df99718fde6245997bdf088fb6086442a883ddb9ed03e3 libunistring1.0-1.0-1.fc38.x86_64.rpm -805da27b46f0d8cca2cf21a30e52401ae61ca472ae7c2d096de1cfb4b7a0d15c libusb1-1.0.26-2.fc38.x86_64.rpm -1a3ce5232d21b6f41c4bf290768684f2e665ca600d95eddfba4c4ada02854b86 libuser-0.64-2.fc38.x86_64.rpm -8ad1a4a44f1052c66318ca789042bedf43d7eea6282ab7872bfecd693b1393a0 libutempter-1.2.1-8.fc38.i686.rpm -c5c409a2d5f8890eeab48b27b9f4f02925a6bbabeb21ee5e45694c7c9009f037 libutempter-1.2.1-8.fc38.x86_64.rpm -b1a0a577a7d03111397b172c63ccf5f22bff39d1e97487d8f6962afc704020ed libuuid-2.38.1-4.fc38.i686.rpm -876ef0556ddeca2c8f56536c80a2f6e0f64357f40bacb92f483adb8a0ff29af2 libuuid-2.38.1-4.fc38.x86_64.rpm -79f80f95acb8fceb4beaa6c1343bc07e9cf6b691ec0a77b79c82a2e74c5845f6 libverto-0.3.2-5.fc38.i686.rpm -292791eb37bc312e845e777b2e0e3173e2d951c2bfbbda125bc619dced7f40bc libverto-0.3.2-5.fc38.x86_64.rpm -39851f80ad4890169e7979b248744acba5cadbc46c58e8e0316aaca1fcf386ac libxcrypt-4.4.36-1.fc38.i686.rpm -52d90d6fbd1ad93f81aad2c4aa38aa52b1c1c94b83560ede25c91b5542da97a4 libxcrypt-4.4.36-1.fc38.x86_64.rpm -e5befc91bfd39ab624cf40c8a1ed32caedc0f74f9ea4fb913e4d9915c1d708c6 libxcrypt-compat-4.4.36-1.fc38.x86_64.rpm -06745f933cdacd7ba3ce77a88016d2a16a1d1887a35b09fe97e574b5743cfa25 libxkbcommon-1.5.0-2.fc38.i686.rpm -507ffdb912296768699a70c30169077b531b1612e47041551bfe523a4b7b6c7d libxkbcommon-1.5.0-2.fc38.x86_64.rpm -be808a1034e6cbf1bf789c960c3e9932b256006d3c29cd18ac3e52fb9f636377 libxml2-2.10.4-1.fc38.i686.rpm -13f2ec62e10333000a13123a4cae5ebbda270c32ece03247e45bd2b244e7bba5 libxml2-2.10.4-1.fc38.x86_64.rpm -741fcbe3e5bdbcf59d90c2f89520dc4001ac4816b57c5c77f6e8a06b53d119ab libzstd-1.5.5-1.fc38.i686.rpm -7d9a98372505c9c1dff7dfea558b20a44820fda416a609467790577a848de110 libzstd-1.5.5-1.fc38.x86_64.rpm -27958b2623e06faf37e427fd4c9750a7b9df35ce38365a93caae068d24ebc95b linux-atm-libs-2.5.1-34.fc38.x86_64.rpm -f0a48ec36269d83120425b269e47ba5c86d5a9a44e0de2665c1d55c10732d25b lua-libs-5.4.4-9.fc38.x86_64.rpm -52898293cf358e998ef24ea977d6381b0a7229ac050f1c4599c5c25823256f7b lz4-libs-1.9.4-2.fc38.i686.rpm -96a8f495896c0ff7520c2cc5c9c173d134efc9ef6c6b0364bc7533aefb578d41 lz4-libs-1.9.4-2.fc38.x86_64.rpm -36a1f0412e495e618ccd8636de3dcac9ad37d5d4c287a1acf2c9af4baa7745e0 memstrack-0.2.5-1.fc38.x86_64.rpm -d5e2e8aed96b57db482e556f593efec98a3238c3a697904eb2eeaf2df7ac4d9e mkpasswd-5.5.18-1.fc38.x86_64.rpm -d9196608152ec34832cc82d3cefa90748f30c75986f2250d8cd0fabc3d0ceba2 mokutil-0.6.0-6.fc38.x86_64.rpm -22f217f91fc2d2a666304c0b360520b13adde47761baa6fed1663bb514b6faf5 mpdecimal-2.5.1-6.fc38.x86_64.rpm -e7c9b0c39f77c6fdf68ff04d8714c10532907a8a9c3e76fb377afe546247737f mpfr-4.1.1-3.fc38.x86_64.rpm -1c055813f64e964a2647da2c889fedb183d4ff27c8a4f4b0674bdbedaee9386a mtools-4.0.43-1.fc38.x86_64.rpm -8dce127ee00b28925e68e2790ff923b51df24441839f1551f30b60b3ea642a9e nano-7.2-2.fc38.x86_64.rpm -9a9dde9e280b31ac75e38051782f72f9dd6ea6077c04e0457b087658140895c5 nano-default-editor-7.2-2.fc38.noarch.rpm -602145f27fd017858256c6ee880863ef5be17c6d3c6c1354f7f16f6f6348db57 ncurses-base-6.4-3.20230114.fc38.noarch.rpm -31f5f49aded8a40f1bfa2be77ffb310785d0e3e3ccff1937e762027cb2add12d ncurses-libs-6.4-3.20230114.fc38.i686.rpm -6ce309d9fd208bfff831981ee4298ccb25fa72363cb7464f1da03b8214d4351f ncurses-libs-6.4-3.20230114.fc38.x86_64.rpm -d1e081f22eade2aea028e152c6690089f44fa71dc536ecccf7f57dabedd0bebd netavark-1.9.0-1.fc38.x86_64.rpm -605d6710ba42104ce0434bb37b0ca9a922a8392c14175bc782f8acb70b94c3aa nettle-3.8-3.fc38.x86_64.rpm -1875738a010be9b1d593f02a95d14afba3742e19354641b851d00b6aaac78246 nftables-1.0.5-2.fc38.x86_64.rpm -c9e8b62c6af7a60a505f881d3cc35294d8b4f51c671c05401133b02ab229c2a7 npth-1.6-12.fc38.x86_64.rpm -db4c16743be18e806ac6e64577a981b227c686ba520161231e52c57831b0188c nvme-cli-2.4-3.fc38.x86_64.rpm -18139e4f2093499d384a1b8c80d51b661c252e4039c1235a6761a280d3260543 openldap-2.6.6-1.fc38.x86_64.rpm -be8a3e233f7a19d84391f2d42f71276cc2053702635e00089b2cbe97372cff18 openssl-libs-3.0.9-2.fc38.i686.rpm -dab96630b0d442164025469b4dce3eccccd482e76ad8ae6f6392045eae147f54 openssl-libs-3.0.9-2.fc38.x86_64.rpm -9cec86553d3cd2facd166d5699fad9624d9b43a39da6fece2d54146bffae5e4d openssl-pkcs11-0.4.12-3.fc38.i686.rpm -cfa3d6feba480abdeb425bc045b525c641c7a864625b1864c2f5721903e364d8 openssl-pkcs11-0.4.12-3.fc38.x86_64.rpm -06d2101874ea4d14b4c73131c5c359d1a2e0ebe0c36a501250026e7b867a0a86 os-prober-1.81-3.fc38.x86_64.rpm -bf2316f23c31d27eeb86855d7c85bc1a696605156a4104d68079f43f7dbe60be p11-kit-0.25.3-1.fc38.i686.rpm -d64a3c3aeac4056185551050ca5eaac2427fc161d93015c20f0012cb6bab53f5 p11-kit-0.25.3-1.fc38.x86_64.rpm -dea697370ede1848c1a54fdccebf792155d98cbdc5de89e85bbc75ec7c94de8f p11-kit-trust-0.25.3-1.fc38.x86_64.rpm -065b99f3541fd5f1281be2082b77e48b835a591776e92f2327bb0462c67baed0 pam-1.5.2-16.fc38.x86_64.rpm -21c59eeb1ad62c09aadca6a4168f927ff943f82e4f764d589f5acb2ab6efc993 pam-libs-1.5.2-16.fc38.i686.rpm -63e970f7b3f8c54e1dff90661c26519f32a4bf7486c40f2dd38d55e40660230e pam-libs-1.5.2-16.fc38.x86_64.rpm -8d846f866158409c775656b39e372d59cf224936d29972d3b6d14e40d3b832ca parted-3.5-11.fc38.x86_64.rpm -c9f6dafbce69557fa0db9291c8328832b7b5978c000b2001a5c3615d03bc859b passt-0^20231119.g4f1709d-1.fc38.x86_64.rpm -43603df046850c4cf067960d8e47998de5c33955b1f865df8d66f20c1b7f676a passwd-0.80-14.fc38.x86_64.rpm -f2737b94fa026a56c7a427f8f4221ff379ea4c4c32f2fff9d95a7a7836dcc6c7 pcre2-10.42-1.fc38.1.i686.rpm -cb1caf3e9a4ddc8343c0757c7a2730bf5de2b5f0b4c9ee7d928609566f64f010 pcre2-10.42-1.fc38.1.x86_64.rpm -756f64de1e4673f0f617a9f3f12f74cceef5fc093e309d1b1d5dffef287b7d67 pcre2-syntax-10.42-1.fc38.1.noarch.rpm -48efa34ce50ae936ab9fe437aa59396d4557ff39fa22cf36c5460d3a986e502f pcsc-lite-1.9.9-3.fc38.x86_64.rpm -aa02afed121e9f5fa9590d75c0b237b6c497ae58c91e0022844b38f594feaeb7 pcsc-lite-ccid-1.5.2-1.fc38.x86_64.rpm -07dc5536982278f38c89517465384ef9f376cd27f0b200806268723993da01ad pcsc-lite-libs-1.9.9-3.fc38.x86_64.rpm -e7509cf0ec99ce89e8e88e9352f733eb9ad14a9c77e0bbfd64826a3de0e4a150 pigz-2.7-3.fc38.x86_64.rpm -e521385a42b3350c0d17e3cbddc0b69c9cf4052d1b77cc8bea2179e05b7d374a pinentry-1.2.1-2.fc38.x86_64.rpm -47178056cb35b3a06a6bbc80eeeeffcf840bff65cd70a0a346eb7364d2917554 podman-4.7.2-1.fc38.x86_64.rpm -440fc5c6e6a37c47f13d1fb53a03f5cb0155592a5bcf9312e2d083d4bed0ad40 policycoreutils-3.5-1.fc38.x86_64.rpm -716096df1b34d768c3e6a5985de8e1ee58b2183ad9f987aa754e592bd2793c70 polkit-122-3.fc38.1.x86_64.rpm -56705b6a1526960d534b0d3e4247deb4eef2b5fa64ceb03544281b8e9bdc4597 polkit-libs-122-3.fc38.1.x86_64.rpm -7ffa0438229228bf5ba18945936d52c3620c95f4a3ffc5c5f0f8774fececac0a polkit-pkla-compat-0.1-23.fc38.x86_64.rpm -fb3fabd657b8f8603c6e19858beb0d506cf957bbca2f3feb827b64c94563b31f popt-1.19-2.fc38.x86_64.rpm -3d5eeb8914b509cebcdf9eb67a70b701727b0b9a77078cd5f6379d751febb969 procps-ng-3.3.17-11.fc38.x86_64.rpm -8b3f681cd05e071d4c7b21eff4684a3ca7674599ee984cccd6a69a685eb8a41c protobuf-c-1.4.1-4.fc38.x86_64.rpm -6983318d6b2dfd4eea29448e9853b74b1d009ab37be7add3ff304ff0483714cb psmisc-23.6-2.fc38.x86_64.rpm -d27890df4880c00b3358caba4d65e6bef5800c1b14c9570280c494e3faa61023 publicsuffix-list-dafsa-20230812-1.fc38.noarch.rpm -e59d71a66652002e1bc6331db17a061bd3ceacf1a449be8af9f7cefc50af4ad7 python-pip-wheel-22.3.1-3.fc38.noarch.rpm -7417816bd96d7b49e5a98c85eba313afaa8b8802458d7cd9f5ba72ecc31933e3 python-setuptools-wheel-65.5.1-2.fc38.noarch.rpm -2f82cfcdf1d58f5e6f279da5324bebafc29bbbdd71d20d11a72d3f1c81f0a0fc python-unversioned-command-3.11.6-1.fc38.noarch.rpm -5c1a9224b32d54fc98cafa86f727009189137e3a4a8b5ecdc831878be85f4ca5 python3-3.11.6-1.fc38.x86_64.rpm -ef27ddd1283ffac1725f343c582c10a0aecbf87d859dbd9328821240ca124b8c python3-libs-3.11.6-1.fc38.x86_64.rpm -ef2089b02db4ade6c0586f05d322d4913a9871442e471bce7f201ea4565ec55c qemu-user-static-7.2.6-1.fc38.x86_64.rpm -2b89bd7b6c38fe28afa8344cf23472d81503cd2fd0be95932203ec4ddb908086 qemu-user-static-aarch64-7.2.6-1.fc38.x86_64.rpm -7d59284568d4af5abaaec73b569dd5ba4beceffecda160208b1711a3eb30022c qemu-user-static-alpha-7.2.6-1.fc38.x86_64.rpm -8f16229370cbb4f7975b95c06bd932f0e9e8d2e533bdf61a7bd7a01b6e7bf8dd qemu-user-static-arm-7.2.6-1.fc38.x86_64.rpm -85b0a4c37d133de10dd7b1aa935885a4f80c68313c7bc753e0acc22d9db81ed5 qemu-user-static-cris-7.2.6-1.fc38.x86_64.rpm -70338be99d5bb1b71b6782bab9ce5abbf68551ecb5518a477df2d10f9cd479c2 qemu-user-static-hexagon-7.2.6-1.fc38.x86_64.rpm -c4411ca439912e850ef5d1e8d3560414b12d11fb29347aef8f633ae2dcef25a1 qemu-user-static-hppa-7.2.6-1.fc38.x86_64.rpm -492a7621120118432809d266dcdf09e492fb3ff94812c21d3ee921d914ce6040 qemu-user-static-loongarch64-7.2.6-1.fc38.x86_64.rpm -36d4dfc57af6bfd985b2da1b75cbd5d9549f011afddcd78af932884d07d931d5 qemu-user-static-m68k-7.2.6-1.fc38.x86_64.rpm -ae146f77d1a81587846035e585df54325a6b4113e7786275053febf8e06d6840 qemu-user-static-microblaze-7.2.6-1.fc38.x86_64.rpm -590bd24d87bd4a690cbb54588aa819a76717a1d86b338731ce0880e5840710f9 qemu-user-static-mips-7.2.6-1.fc38.x86_64.rpm -a0808cd9fbf91a18f8a683183dc7221c27c3fe968870f56f7b4d1beed7d8341c qemu-user-static-nios2-7.2.6-1.fc38.x86_64.rpm -5bcf7d28465efb51e1b88b02dc3858b8e3d8db3b1507d92382d86858dd5c4658 qemu-user-static-or1k-7.2.6-1.fc38.x86_64.rpm -0f25489b23695395421d1826ec159faff75b3c259595a1c49b458779d7933e52 qemu-user-static-ppc-7.2.6-1.fc38.x86_64.rpm -b2135034b08f4c26bc14b56beaabe56880d6e4c34ccf5152f6841762f2ce7465 qemu-user-static-riscv-7.2.6-1.fc38.x86_64.rpm -0da8ee9922ed80e84614049f43919c492b361f2187f04fc0dcc386ad0ff87c32 qemu-user-static-s390x-7.2.6-1.fc38.x86_64.rpm -4581ab68c205d9d990923b5ed17471461470db8114e8fc50a9a5307ff9e1e140 qemu-user-static-sh4-7.2.6-1.fc38.x86_64.rpm -58bd2bd48a578035af82792b1520c2d8d79bbfc2432100a49ca74865d75279df qemu-user-static-sparc-7.2.6-1.fc38.x86_64.rpm -ea7664d20d64a2e7ed15bae3d161622bb448d5fb1f979f483ae089daff8d37cc qemu-user-static-x86-7.2.6-1.fc38.x86_64.rpm -d730fc60accf0e79deab6bca815fcde11afd4143021f5fcdac4e654d0aa96d1a qemu-user-static-xtensa-7.2.6-1.fc38.x86_64.rpm -0de622e220594daf822dd0015e099b9940b5d92768703ead1b53e1daf06aecf0 qrencode-libs-4.1.1-4.fc38.i686.rpm -49ec489f168c1671a2babb690edfb020a5252f38e8d0b2d96465070abd2b0d70 qrencode-libs-4.1.1-4.fc38.x86_64.rpm -fd681c9e466bbed2411ff931b80bf46f04279b001fe8578abada13952854a774 readline-8.2-3.fc38.i686.rpm -a7099c322c45030738bdd90e3de4402c0c80c6ebd993a1749c2e582cf33ee6f2 readline-8.2-3.fc38.x86_64.rpm -ff3ac983386ce4bc4ce48dc67bb0b50b8b078a35ccc6897597b82852a648bdf1 rpm-4.18.2-1.fc38.x86_64.rpm -a30278c06ac5aa081864793a3960f7d2701f0d7f0b594724d9aec7b127f5b758 rpm-libs-4.18.2-1.fc38.x86_64.rpm -1e8e041e2fe2bccbed82917515c8605c2ea1947dcfed95377f7f7b3038e0921b rpm-plugin-selinux-4.18.2-1.fc38.x86_64.rpm -b4f4c54d4639ba007f87b95fb7ef1819c5210a996c2377f1ebeb729ebda0324c rpm-sequoia-1.5.0-2.fc38.x86_64.rpm -39f9925ce9c0e5133040a10d043e3e6d22cfcfdfef3d3161f2244c98b555b352 runc-1.1.8-1.fc38.x86_64.rpm -61985efd54550e1fa9f31575eaa57232034d72d4eb169e88e70183d43727962c sbsigntools-0.9.5-1.fc38.x86_64.rpm -a6e01b89e814ec42d1c2c6be79240a97a9bd151c857f82a11e129547e069e27f sed-4.8-12.fc38.x86_64.rpm -3f7bfabe71024d3678032c639c980064909749fda66c0d4b5b39566c454a03d5 selinux-policy-38.30-1.fc38.noarch.rpm -0b9b36866d5edcc8d9afa4d52909ccb89fc420860c5a5557c692c96e23404203 selinux-policy-targeted-38.30-1.fc38.noarch.rpm -c7efb8634b62cdab9e8894174a8c70d20eb431482277231bc54fa8ca8c3681e3 setup-2.14.3-2.fc38.noarch.rpm -8be96e09e2e44491b287be44b2b6be0c9f8aeab75fe061d3e8a28b9be19144ef shadow-utils-4.13-6.fc38.x86_64.rpm -46eaa576b2c5fcd167d9d88f823c309da0067fa8c4d35b7c535792fe862f05cd shadow-utils-subid-4.13-6.fc38.x86_64.rpm -8fc0ae5f47a2d868bbd44c5e65d68cec44272e6e7bbdc506217fa5c66499f872 slirp4netns-1.2.2-1.fc38.x86_64.rpm -868aa887826ef8c81162b97ed4440949741ecb0d78bba12291b03bdccb917877 socat-1.7.4.4-2.fc38.x86_64.rpm -be6d9e9b98733494ee5901d45b849e2dc012a6b41c3ff09d0d212002dbe15dce sqlite-libs-3.40.1-2.fc38.x86_64.rpm -580dc4554d6f353b00238fa68d6aa7af6d8d4fa3b408ceb3c5092fca36e04b36 systemd-253.13-1.fc38.i686.rpm -088a4c0a7572ebf39ff32bd299dd86981339e0531504a2784e22a49d467bc498 systemd-253.13-1.fc38.x86_64.rpm -6f6ea4258c2a8081e3b76064cb61b8147b3555f11181c16385b4ef9b05ff54df systemd-boot-unsigned-253.13-1.fc38.x86_64.rpm -8bf863e256ac017761b69915ad4f94fd445499d6417ffc518409dc491d35599a systemd-libs-253.13-1.fc38.i686.rpm -9ee15fb0346f334335453cc2092fb3aa77ae0961cb54590d2e374e07a488d4e4 systemd-libs-253.13-1.fc38.x86_64.rpm -3436c9ce186df4d0aa51eb3f91e138357fb2a04abd07bb10a14f6fe05682f062 systemd-networkd-253.13-1.fc38.x86_64.rpm -e772094969a9073b1daefdc75354f769276767f4e0ae953f54554ccfbcc8e375 systemd-pam-253.13-1.fc38.i686.rpm -0761b5a2095acd4b39d25729f3f0cd793a99c96b01a5b2d0d285825021a553f5 systemd-pam-253.13-1.fc38.x86_64.rpm -169e6a40322674ebf58269a8627d14f16dd657afd0e8a654a3514da0389cd16d systemd-resolved-253.13-1.fc38.x86_64.rpm -8d37101023d15a205164d44f0ffd671d9215493bc4f854d2b989fe4102b69201 systemd-udev-253.13-1.fc38.x86_64.rpm -1d6caa060ef12ab32bf7220b49bc0d9c858c68a8f50b060f5117a2aca63a4dc5 tar-1.34-8.fc38.x86_64.rpm -5cc364cad8cb613a0ec68076f7f8e10b72ef2e266a10d2463bf548b2b0abd13e tpm2-tools-5.5-3.fc38.x86_64.rpm -8adf29af85920514902bc4332ceb896a54f9cf89e08993c9345b62c4140f91d9 tpm2-tss-4.0.1-3.fc38.x86_64.rpm -041e8b9be8a87757da8d5b8a2ced4b5aec8bcafd1c0747234cdfe10206eae27b tzdata-2023c-1.fc38.noarch.rpm -232da16c546617adde46ecaa1d5367acd05f75d04570fb367123b8dd01abdea4 util-linux-2.38.1-4.fc38.i686.rpm -f0f8e33332df97afd911093f28c487bc84cbe4dcc7bb468eac5551d235acee62 util-linux-2.38.1-4.fc38.x86_64.rpm -b57dbbbee14301e89df618b398ef39b7fc841eaba6be1b6346cf37ed7695c26a util-linux-core-2.38.1-4.fc38.x86_64.rpm -eeaffc0c8f3bb07feeb1dd8be415463f140ce5e7b4d64093c5dc6d6ce9dfff8f vim-common-9.0.2153-1.fc38.x86_64.rpm -05df0d7a7bbe512b35c77a1b74d9bac791ab3370a2c3c656a8f0a50fc0482afd vim-data-9.0.2153-1.fc38.noarch.rpm -7105ee08f1f8e839b112d1457a8b8259ed2260850a9fd38f5f1f64279c8f87cf vim-enhanced-9.0.2153-1.fc38.x86_64.rpm -2e8e08df6fbb33e5c71fc3a814d64d98803f88b7ba911fe5371d224d95c40e63 vim-filesystem-9.0.2153-1.fc38.noarch.rpm -7f8524d182dacd6bef744c11d225dd63a82100350e95fe3ec414e70cf642c1f1 wget-1.21.3-5.fc38.x86_64.rpm -2c8b143f3cb83efa5a31c85bea1da3164ca2dde5e2d75d25115f3e21ef98b4e0 which-2.21-39.fc38.x86_64.rpm -84f87df3afabe3de8748f172220107e5a5cbb0f0ef954386ecff6b914604aada whois-nls-5.5.18-1.fc38.noarch.rpm -59a7a5a775c196961cdc51fb89440a055295c767a632bfa684760e73650aa9a0 xkeyboard-config-2.38-1.fc38.noarch.rpm -8b14b303b26c8aad75c85a3461f961785cf6d37ccd325a05dd8e9e24e1e4042b xxd-9.0.2153-1.fc38.x86_64.rpm -e911703ffceee37ec1066344820ab0cf9ba8e43d7957395981ba68c4d411a0a4 xz-5.4.1-1.fc38.x86_64.rpm -2b3a57c5ccfd4c99ec78d8420394387782a4ac57946d63800a406a4050c3d214 xz-libs-5.4.1-1.fc38.i686.rpm -bfce8ac2a2a78a23fb931531fb3d8f530a78f4d5b17f6199bf99b93ca21858c0 xz-libs-5.4.1-1.fc38.x86_64.rpm -e6971389d89ab454bbb372859b5aee67469a0b12e57d8657c0818bca78df22f2 yajl-2.1.0-21.fc38.x86_64.rpm -c83464d6c93835b24b2c09d4c851f4a9bdacc70489a6c21447ed33ffd98c0d63 zlib-1.2.13-3.fc38.i686.rpm -c26d4d161f8eddd7cb794075e383d0f4d3a77aa88e453a2db51e53346981f04c zlib-1.2.13-3.fc38.x86_64.rpm +37abef83e8927b4b48f69fcbdcc249d349c6029cc669401676d01f0ea326999e WALinuxAgent-udev-2.10.0.8-2.fc40.noarch.rpm +03b5889fecc19101fe5c5eb6105bded5705e16a7af957f92b6cc6a7a96e829a8 aardvark-dns-1.14.0-1.fc40.x86_64.rpm +ac860c52abbc65af5835d1bd97400c531a5635d39bc1d68e36a1fe54863385ea alternatives-1.27-1.fc40.x86_64.rpm +b28e7d90ed3aeb6ca10ecb235b70534665011af35bd3677fb836b1e3cfa602a7 audit-libs-4.0.3-1.fc40.i686.rpm +accc1c623cc5345f983990416ad0f8d6028d63bc9f00bcb808688b2a3bb7caf9 audit-libs-4.0.3-1.fc40.x86_64.rpm +e9fd8dd4c9068501c169edb684de7f6e38657548e9d4d1b838a4d6316e9f17cc authselect-1.5.0-6.fc40.x86_64.rpm +db18a583ebde21d8b0b67f0306e25908b273bef9c532469ac0b7ab92578438f4 authselect-libs-1.5.0-6.fc40.x86_64.rpm +6404b1028262aeaf3e083f08959969abea1301f7f5e8610492cf900b3d13d5db basesystem-11-20.fc40.noarch.rpm +156e073308cb28a5a699d6ffafc71cbd28487628fd05471e1978e4b9a5c7a802 bash-5.2.26-3.fc40.x86_64.rpm +208ddebcd5edbff3dca54020a8a180f5410ea9b2d82c733e106992a729b4b84e bzip2-libs-1.0.8-18.fc40.i686.rpm +68a43532d10187888788625d0b6c2224ba95804280eddf2636e5ef700607e7d0 bzip2-libs-1.0.8-18.fc40.x86_64.rpm +1afcf80d5e7b22ee512ec9f24b4f2b148888ef95af3486cf48f2204c3406b12d ca-certificates-2024.2.69_v8.0.401-1.0.fc40.noarch.rpm +a9af8b72483b4ad6489e6d2492161120809453373edaf24722cffb394bd1cd15 catatonit-0.2.1-1.fc40.x86_64.rpm +588a2ab4dd93d58ca8b8d2c2d0b5e2c52007548c3fdd06f5ca1ab415ee236d86 composefs-1.0.6-1.fc40.x86_64.rpm +5935816e8d377d0385e5287ca12e4d3b43e3c3cdc9cc4deafa653a6dba78611a composefs-libs-1.0.6-1.fc40.x86_64.rpm +db246f6445469b5a71e965a081685471768393cf04181e7250ce0ddcb8a9c3d4 conmon-2.1.12-2.fc40.x86_64.rpm +adf4b75cdd9fae9d2d37fb71d9f0bf625a6705c0f0a7784569ab21463fe22152 conntrack-tools-1.4.7-7.fc40.x86_64.rpm +b84841f98b3c14d68da5a4021f10973eb5fa6566e63f4933e40a0c91aa134ce4 container-selinux-2.236.0-1.fc40.noarch.rpm +bbe29e0c7b4ca076d50b4ac3954eb383459230d96b13f353ee71ebd5de33b6d1 containerd-1.6.23-5.fc40.x86_64.rpm +46f035201c84f65f6b9fd549adbe1c4aeb5f5972e69bf6537c05e7cc20587a4a containernetworking-plugins-1.5.1-2.fc40.x86_64.rpm +8c6676fd3d9cb966e68da46dc8bc3a81ea6f2ccb697f75a284a9c9b868489789 containers-common-0.62.2-1.fc40.noarch.rpm +70956b80c95cd582f6d5410c555a1f7468b13c1f022bb25856a6a1a59f4279e3 containers-common-extra-0.62.2-1.fc40.noarch.rpm +299d3e7e1cbc110d9ae8a47f6ca95142c3e3783cb1464bfbd6bc550c414b97ec coreutils-single-9.4-9.fc40.x86_64.rpm +d941a78ffb6e2e0b4c24d0097d0351ced8796edde90208b4bddee459bce0a949 cpio-2.15-1.fc40.x86_64.rpm +faa23cb6a7a612c0a6e874c788c5add967c5e193bd38c2e6093b82b38a162f81 cracklib-2.9.11-5.fc40.i686.rpm +ea1f43ef9a4b02a9c66726ee386f090145696fb93dff80d593ac82126f8037ec cracklib-2.9.11-5.fc40.x86_64.rpm +fe24641e69545c428890a4b094f015c03f65a6c30c3db7bb0de7672bab66bfd6 cracklib-dicts-2.9.11-5.fc40.x86_64.rpm +cb6318cb928c70696f1fc3a79469c0343905b4b69c5d9789e9932b10b4584357 criu-4.1-2.fc40.x86_64.rpm +05a1dbc9f2e3585df8f9930327a7e6f7f59b396359db92787086e1fdb73634b2 criu-libs-4.1-2.fc40.x86_64.rpm +0a8ee60884b6739bc0fffed6c47a94eae524e73cbd9c942420f6ffdfcd39086b crun-1.20-2.fc40.x86_64.rpm +d7a62ff0193375607d28d8fe7eedf3ff5b6ddac154e1474d79787b9f32ae298d crypto-policies-20241011-1.git5930b9a.fc40.noarch.rpm +0f1d436f879fa30f18adca576a1f91bb1a8d1ac42cff5f35c1d15e6dffc506fa crypto-policies-scripts-20241011-1.git5930b9a.fc40.noarch.rpm +26aadc06a9f98c58ca6250d811e749ee5fa76059b37445ec28b50ee73d548174 cryptsetup-2.7.5-1.fc40.x86_64.rpm +765d5162ad7b1eef16a3d3e4285743a3a358552112e7946dcd974e355dc9fa28 cryptsetup-libs-2.7.5-1.fc40.i686.rpm +6559b4d59d898317972e8d728253ba4315578c15fe23553958d6fb8033698794 cryptsetup-libs-2.7.5-1.fc40.x86_64.rpm +20b0f2923feae4c2f1d339e959d3f03d81f8ca985faa05872377b827d6f30467 curl-8.6.0-10.fc40.x86_64.rpm +0dff67dfeca59cb68cadafe8d9909b88dfaa2fc0a9a4426352f66a5fe351fbe3 cyrus-sasl-lib-2.1.28-19.fc40.x86_64.rpm +19197df26f76af5e78bd1e3ad2f777bea071eef6dfec1219f6b8ee3c80e10193 dbus-1.14.10-3.fc40.x86_64.rpm +84ca6055aa354df549fdc78d6d9df692ed4d12c14a489a6d2ce844b5f225a502 dbus-broker-36-2.fc40.x86_64.rpm +81bade4072aca4f5d22be29a916d9d0cfc9262a6c5d92ddfe750f7b8bf03f7c9 dbus-common-1.14.10-3.fc40.noarch.rpm +51bcbc8f98a8abea9a24f3f3988958b965dfe8d6b4201ee046255416fe3ce75c dbus-libs-1.14.10-3.fc40.x86_64.rpm +6a2046c5a6c959cdf30519ec67385faf9b6156ffb3b449c54a0694d4d35985fd device-mapper-1.02.199-1.fc40.x86_64.rpm +33d2dfe3cf355511e6d7bbd1969675b9b73f58bac2a998e9d950ab99d762a299 device-mapper-libs-1.02.199-1.fc40.i686.rpm +f80de90bd6ab88c9071b200c0f0323f80145ca3ab9ac565e8f569390890137cc device-mapper-libs-1.02.199-1.fc40.x86_64.rpm +6913a547250df04ec388b96b7512977a25ab2fca62ed4345c3a9fc8782ce659f diffutils-3.10-5.fc40.x86_64.rpm +cb0736689bd171b6c6ac7a60737fd6b9534c950958ad8e03138068bf9498e0b1 dracut-102-2.fc40.x86_64.rpm +fa40cda554dc644d5a8354b18be748f21996dadd6193ee4ac32c02581266d313 duktape-2.7.0-7.fc40.x86_64.rpm +ac4f1b2eaf5d452512e7b6172c93880c2b501946b71a228adc02d50bb3fb56e0 e2fsprogs-1.47.0-5.fc40.x86_64.rpm +8476fda117e3cb808129ddc2f975069685a8c7875ee04c3dafa6ceed948a2628 e2fsprogs-libs-1.47.0-5.fc40.x86_64.rpm +2e2bf662f060ddd75195e9d3d5f08cdd5d9cc857df3a9bcc45608337ba314a25 ec2-utils-1.2-48.amzn2.noarch.rpm +e6231ec4268b3efa928250eb4106311e0f33396422245b938bfed4ba2d79c573 efitools-1.9.2-9.fc38.x86_64.rpm +6ac676d78c2df896f9794a8dffb75ea69c58d202c68f4bcf084f0d264154a666 efivar-libs-39-2.fc40.x86_64.rpm +75a77fa962df4aabee266a1a48aeb4bf3a164dba31a597af23ab33c693a1c068 elfutils-debuginfod-client-0.192-9.fc40.i686.rpm +4b4b9b5c7e4ceaf65cb473089f87eef07cbdb7254425f60219c5b6412ea7da94 elfutils-debuginfod-client-0.192-9.fc40.x86_64.rpm +633b68d0b697c585727d07f4a3c5e4ba536841a8717eaadc552fc10e61d3b86a elfutils-default-yama-scope-0.192-9.fc40.noarch.rpm +fde3769dc677f0a76ce96058c3032f6553809f4809509842b9523a13e90913e2 elfutils-libelf-0.192-9.fc40.i686.rpm +f7778242b3242fab35a5e0c4f0919ca9823a47c7b296f274ff312cc2c49ba6e7 elfutils-libelf-0.192-9.fc40.x86_64.rpm +a322c6f44f82bc21558f3bb7afc694dbdc28a41340280d3a2c5da2656a339d2f elfutils-libs-0.192-9.fc40.i686.rpm +184235133a6873a42b72d2d88657f12dd4b4b603fd5058fbd02daa0a7308f69c elfutils-libs-0.192-9.fc40.x86_64.rpm +a09546cd7e26c630765f4acb93942a2c70524107a50e20761eb0898b8a80b6cb ethtool-6.14-2.fc40.x86_64.rpm +136ed00bff4d44ab7bd6d0926c4b80c2f5ee9160dc4f698be988ca3d37303b4d expat-2.7.1-1.fc40.x86_64.rpm +849feb04544096f9bbe16bc78c2198708fe658bdafa08575c911e538a7d31c18 fedora-gpg-keys-40-2.noarch.rpm +8f0a56982aa10b607bd3b8009e91bac1a7d9b7ba40eb4435ff9bd9efe7e5e76e fedora-release-40-40.noarch.rpm +dde6f4b5d4415ce20df40cf1cb9ff3015aa5b1896c5b5625e49aa686cdce1d1d fedora-release-common-40-40.noarch.rpm +f133aabe97d4aff7e6c83b8d364fb9805f655b32150e2f6e876958966293e467 fedora-release-identity-basic-40-40.noarch.rpm +e85d69eeea62f4f5a7c6584bc8bae3cb559c1c381838ca89f7d63b28d2368c4b fedora-repos-40-2.noarch.rpm +a6f2098fc2ed16df92c9325bd7459cc41479e17306a4f9cddfd5df8a1b80d0f8 file-5.45-4.fc40.x86_64.rpm +f76684ee78408660db83ab9932978a1346b280f4210cd744524b00b2e5891fe1 file-libs-5.45-4.fc40.x86_64.rpm +063af3db3808bea0d5c07dbb2d8369b275e1d05ad0850c80a8fec0413f47cd64 filesystem-3.18-8.fc40.x86_64.rpm +21725de2a93e1ea19f8d298e32a2428a3a08b9c98f22561cc778a807ed43639f findutils-4.9.0-9.fc40.x86_64.rpm +f4c2d51c7b4577f7b7ef498f8e2afb1b007da2de00cca28e220f50129c40a48c fuse-common-3.16.2-3.fc40.x86_64.rpm +f94315e447afb7442033b7b82e43a4ed62754f603afda53930280300855e46c7 fuse-libs-2.9.9-21.fc40.x86_64.rpm +8fe84b7e0319afcc9c9eb28130b74e0cd7c675667a6ce075eb7ee2ec1b0014c2 fuse-overlayfs-1.13-1.fc40.x86_64.rpm +2d6631d65e3b5c91afdb100a51ee8e50294f0e074a944c1662008d878d47456e fuse3-3.16.2-3.fc40.x86_64.rpm +a9c6502a5b190aaf169e93afd337c009e0b2e235e31f3da23d29c7d063ad2ff9 fuse3-libs-3.16.2-3.fc40.x86_64.rpm +6c80dfdaf7b27ea92c1276856b8b2ae5fde1ae5c391b773805be725515fdc1ac gawk-5.3.0-3.fc40.x86_64.rpm +c4cc69bf3a2655b9ee9ac23492d377bac57811c5b4f81fbf43537520ee33c7af gawk-all-langpacks-5.3.0-3.fc40.x86_64.rpm +21470eb4ec55006c9efeee84c97772462008fceda1ab332e58d2caddfdaa0d1e gdbm-1.23-6.fc40.x86_64.rpm +93450209842a296ea4b295f6d86b69aa52dd8ec45b121ede0d5125aa49bad509 gdbm-libs-1.23-6.fc40.x86_64.rpm +40337d9167737abe23af9c6b586b883f33cc82028d69b29c1d68b524201d9248 gettext-envsubst-0.22.5-4.fc40.x86_64.rpm +554a68e692ccdd0cf71ea67a4c550bac910685465f17eee503732d48ccda9c90 gettext-libs-0.22.5-4.fc40.x86_64.rpm +046971e9f5f0c88737854e1c9e02cce8f5854633575984b235cf3f8b11ec7b91 gettext-runtime-0.22.5-4.fc40.x86_64.rpm +6afaddcff936e900b9d3ab379ddac5c8dd0aa323924b973bf2fa9fc819dc3638 glib2-2.80.5-1.fc40.x86_64.rpm +ee01787bf89f4f3b1ee7dc269844bae2fc672b1af4ac5e241e570391fbeeab3b glibc-2.39-38.fc40.i686.rpm +b64c709529bedb9d3e62c892fe79b67f186c499d54d1e7985713ddb5fded7a59 glibc-2.39-38.fc40.x86_64.rpm +5cdec5bdfe58752fca36279d5cd25e08a6b37a086020f2573937304a5b770dc6 glibc-common-2.39-38.fc40.x86_64.rpm +e8ab57f139980b93c17a9598a40cac5a81be37216526be8115235a57ce5a9df7 glibc-gconv-extra-2.39-38.fc40.i686.rpm +4deff99dffc791e504d9e1d9b5c19f3a053a14291d2168ffd33fdcd24180323b glibc-gconv-extra-2.39-38.fc40.x86_64.rpm +8556a19ce4e3a9977f2d1cdb2f1885eef6401140609264e2c50a8161874d22ac glibc-minimal-langpack-2.39-38.fc40.x86_64.rpm +b054d6a9ee3477e935686b327aa47379bd1909eac4ce06c4c45dff1a201ecb49 gmp-6.2.1-8.fc40.x86_64.rpm +0a8b1b3fb625e4d1864ad6726f583e2db5db7f10d9f3564b5916ca7fed1b71cb gnupg2-2.4.4-1.fc40.x86_64.rpm +4425dbd35ab65f25b092d12ac56c4b565371a1c52ac882c8896dbeae7d52bbb1 gnupg2-smime-2.4.4-1.fc40.x86_64.rpm +74f185a772b22db0d41b69d63d1fa4a78840bc32359ce47729dbd54d297ffa77 gnutls-3.8.9-1.fc40.x86_64.rpm +9a463e49371c05bce6713fd4956ff3784db265485260e6151d3eba444ad41f42 gnutls-dane-3.8.9-1.fc40.x86_64.rpm +f8510eeec17b9258de9a68ce15af21f3ea135b5e767f3bc9047f851d81dbac6e google-compute-engine-guest-configs-udev-20240830.00-1.fc40.noarch.rpm +94e443590221fb17e0330f076ebac32baab17b8d9c22566db372899ae750ca64 gpgme-1.23.2-3.fc40.x86_64.rpm +6d54af0fc5ae216eb97720415acda4245ebc6c021420a2892b58620b5b25ca38 gpm-libs-1.20.7-46.fc40.x86_64.rpm +8e2310f6cde324576e537749cf1d4fee8028edfc0c8df3070f147ee162b423ce grep-3.11-7.fc40.x86_64.rpm +46bc4d8d62eeb1fa3275898d44a30643260f4dda2d3d2a3992f879e4c7df26c7 grub2-common-2.12-16.fc40.noarch.rpm +c508cb9605baa6a0751feda2d23e9a65a4d11538d6d108c295562eac65c63ab1 grub2-tools-2.12-16.fc40.x86_64.rpm +0c6bb04c6e81d368793c03bab2ec736c3ea63aaea9dbb64d4a94b12b35320bdb grub2-tools-minimal-2.12-16.fc40.x86_64.rpm +6a146fa9b154e67eb67eeb258df37814a863997c87171fdc2bd771e5a46b1cc4 grubby-8.40-75.fc40.x86_64.rpm +6dcc2f8885135fc873c8ab94a6c7df05883060c5b25287956bebb3aa15a84e71 gzip-1.13-1.fc40.x86_64.rpm +7ea61bdaada7c1ab5b8567e054a73e2cb3ca6019e3db887049998fed7eea8514 iproute-6.7.0-2.fc40.x86_64.rpm +21d9bc4c677edd86b2e88ebe4c20b097412c2fa3ef4a91d7de0f9b03e1306f5d iproute-tc-6.7.0-2.fc40.x86_64.rpm +21e1196534fbb6d6a4f8d29a1e76518e2740ae53f63080fc811e7ce9cc6d0982 iptables-legacy-1.8.10-7.fc40.x86_64.rpm +6e78bebc0bde8c2f1bd9c4a5f40010a779e7505a0fe87aab516db1bb4a840f02 iptables-legacy-libs-1.8.10-7.fc40.x86_64.rpm +98115e0aa89bc9a8ec66c160af80cf32519db427a83d23d8f3dd9185b3aa591e iptables-libs-1.8.10-7.fc40.x86_64.rpm +99857ab7055ee14a0b4c3a77d1ee9b46217359c708d1bd4efad348595ea974fe iptables-nft-1.8.10-7.fc40.x86_64.rpm +9b4f2730a62955650c1e260e1b573f089355faf0155871e2c10381316a3b2e55 jansson-2.13.1-9.fc40.x86_64.rpm +3a4b45b9f4746a7d3ce8fcb853fa65563051d2f6f31826b8007f6674fe3d8ce1 json-c-0.17-3.fc40.i686.rpm +77e67991fcd4eea31f5b2844898a7854768548f0ab3abf7beaa91526afbf794b json-c-0.17-3.fc40.x86_64.rpm +c8e382e9de90e6946dd9bc2f706d6c307ea4ebba3eca91a283f1bb72b5b3ac9c kbd-2.6.4-3.fc40.x86_64.rpm +4764830a5f91f668f6348539777ddcbc2aa5c21f433f2ef8fec4c385db26233b kbd-legacy-2.6.4-3.fc40.noarch.rpm +61b2303ebb8e37c240b7cf10a8649dbdd2ddefcb571a4e6ea688cc3ad9168970 kbd-misc-2.6.4-3.fc40.noarch.rpm +547bb1919671ef5b3f7ee40dac55d6d681374fe7367fb7c369b02b6287c25fca keyutils-libs-1.6.3-3.fc40.i686.rpm +387706fa265213dc46e4f818f30333cc93f0c54539cbd2ec4db3bc854077307b keyutils-libs-1.6.3-3.fc40.x86_64.rpm +370582ae0ed0dd607ec38e92eba4b124a4b2ec3835f3f4c1e5ad8255ee11d692 kmod-31-5.fc40.x86_64.rpm +42994ac67877595861b55adafd75ab3ce02d397e2ccddac8fb40ec0fecb4436b kmod-libs-31-5.fc40.i686.rpm +53dd95341767a2ea40b68e4621a231883bd5b69426f0920ce1f1ca94e18765cb kmod-libs-31-5.fc40.x86_64.rpm +9a03b21936528f6d08700757cb460c48e9557a71efaaa5e93b01b3f7614320f3 kpartx-0.9.7-7.fc40.x86_64.rpm +cd3402d654af18c421c0ae866ef668094cff5c032bb3f769606261eca8dcf8fa krb5-libs-1.21.3-3.fc40.i686.rpm +878a5a48835ecfec5fa04c7c7a1f24bdae7bd8e9aeca7b3f9dd97f6a23b9b41e krb5-libs-1.21.3-3.fc40.x86_64.rpm +6f2f0a522f2f10f273a77a60fdb7e066c14059d0a3676c9f723162daa7110b42 libacl-2.3.2-1.fc40.i686.rpm +b753174804f57c3c6bae7afeb6145005498f18ae5d1aa0d340f9df5b8d71312f libacl-2.3.2-1.fc40.x86_64.rpm +74d72760c1982830358d676794ee3972ab05550fe7235ae9756a40de8266091f libarchive-3.7.2-7.fc40.x86_64.rpm +e131ab89604dbd4fdc4f80af632099e48bf68bb328dbf0e7dcbef1d1e134dc09 libassuan-2.5.7-1.fc40.x86_64.rpm +67facd893f5082be270d0887a43ba22492c47e652e06e5d53ecd681a1aec8ac7 libattr-2.5.2-3.fc40.i686.rpm +504cff39c51a04c1d302096899c47dc34ac0eba47524c2fc94c27904149e72cf libattr-2.5.2-3.fc40.x86_64.rpm +649cceb60f2e284f8d5dadeec4af8e7035650fe0e5aa75c552354b3fa5708cfe libb2-0.98.1-11.fc40.x86_64.rpm +46e35f0fcf3c5ac842c4fd7ba63c0106b73fa2250a9ae1f8e92c53d552201d7f libblkid-2.40-0.9.rc1.fc40.i686.rpm +f4278c28f2bb21b0c24a5975384a5ccfc0934504d8a7c036ca1346b7683e1b5c libblkid-2.40-0.9.rc1.fc40.x86_64.rpm +c0f6262d325a8a0609935abcb922033c6e56d6b6e5096ba8fbb9c972817e27d9 libblkid-2.40.2-1.fc40.i686.rpm +b506de64d63262d9d957a75fdf2282d82b1e4978cebbdfc191ef93bba37e3b7c libblkid-2.40.2-1.fc40.x86_64.rpm +d8fab37e62c441e5d35421086a20923ff15561ad28c858cfb670b7c095106dee libbpf-1.2.3-1.fc40.i686.rpm +fca2d942f6264b630b33991e48dcb605543a4c837371f28f92994bf956677f24 libbpf-1.2.3-1.fc40.x86_64.rpm +97e9e5339bb0ca6ce3d0195c8ebe48384bcfc087ee6bc7a35b1d27d4de23fbfa libbrotli-1.1.0-3.fc40.x86_64.rpm +8163fa05deeb06414a6e9ad78ba8d3e65eab9af87f35f0790f8c81bee8359ab6 libbsd-0.12.2-3.fc40.x86_64.rpm +0bdb66863b60abc8c2ca540e80ef58e9d4da3f700b685ecc49042616387dee8d libcap-2.69-8.fc40.i686.rpm +6c92fc0c357964d2b57533a408ec93b7fe5214c1f0b63a6b1c0564b2ba5c481f libcap-2.69-8.fc40.x86_64.rpm +9b11d2f4bd3e8fd56028deb22e86e175c5d91017db81413f108f0841a9a02349 libcap-ng-0.8.4-4.fc40.i686.rpm +dc22477c3ac762f92ecc322af4f39fee2c5371bedc495ce242f9b94c590c580f libcap-ng-0.8.4-4.fc40.x86_64.rpm +274fd72d27570f3fcc9f06efedd21ea7a71e0903c286222fdbbefd6b30b9a80c libcbor-0.11.0-1.fc40.x86_64.rpm +c890a19d2c4a3da836bae1db40b778fe0339cd0d26bddfbe584aaccb1a0f1485 libcom_err-1.47.0-5.fc40.i686.rpm +0d100701976c37fe94e904ed78437db7477ae1dc600ece07bea23fbbd968762c libcom_err-1.47.0-5.fc40.x86_64.rpm +7583f6b188f19e5112403ad53d0e5c98d35a5b736d355ea91d4a35ade5ef014c libcurl-minimal-8.6.0-10.fc40.i686.rpm +e3dc770fc4c48bec2da9ac948bcd43e053608d0397ad0a57056409a7d427289d libcurl-minimal-8.6.0-10.fc40.x86_64.rpm +700d56839e1bc16c08f71c505a7e62f655e4c18f4bf71bf2f36f3854f829e6f5 libeconf-0.6.2-2.fc40.i686.rpm +2ef764049e121ee2a9fa5d0296e6e2dd0abc7541040b8e49d67960bd9bde74e4 libeconf-0.6.2-2.fc40.x86_64.rpm +7a91572e9639617937c13cd103ac5571075f37fa533d796be108aa8a4937432d libedit-3.1-54.20250104cvs.fc40.x86_64.rpm +c4adcee5dd9e22ea50d6c318ac4936a8df708121741958ce5aa8f038c46c61a9 libevent-2.1.12-12.fc40.x86_64.rpm +a1ba3045c99ef1b266383f0801731a68f9e0cb069a6c808267ad33b759381907 libfdisk-2.40-0.9.rc1.fc40.i686.rpm +17f02ca51b90580887d739f52b995034e0929fc6bcd92be308554a2f5337bbe4 libfdisk-2.40-0.9.rc1.fc40.x86_64.rpm +34db4e48052a47e1c6f445dec3edbdb279890ae24c5ecfa44e8a19fdee0a99a2 libfdisk-2.40.2-1.fc40.i686.rpm +aa6a51bbe265bb3d3a50c37557f6513d51298301e4957ce4484e56feb837fa32 libfdisk-2.40.2-1.fc40.x86_64.rpm +25caa7ee56f6013369c2fac26afd3035a7d580af0b919621ba8d495d13a5af86 libffi-3.4.4-7.fc40.x86_64.rpm +f9c5369b6d168a2b8e46159bc41ef0755ee1a8d12f4c6766fdfe23e827cf5cdf libfido2-1.14.0-4.fc40.x86_64.rpm +460a36745833f629ac1f5d232ec0daec092b7cb654a4bf3e4fde7c693fea9fbb libgcc-14.2.1-3.fc40.i686.rpm +cd073c42cb4dfcd224e9b4619883f2c7923ab0b083d7c90b01e3052c89f6b814 libgcc-14.2.1-3.fc40.x86_64.rpm +10c4c12c6539ffea68974cd9b57013d471ac35fe3bef4833c0a22f6b29fbf489 libgcrypt-1.10.3-3.fc40.x86_64.rpm +03d5f4d139dec2e7c94714b1b9f59d37236dbda9f09271bdda99c71251f15f0e libgomp-14.2.1-3.fc40.x86_64.rpm +8d0a9840e06e72ccf756fa5a79c49f572dc827b0c75ea5a1f923235150d27ae2 libgpg-error-1.49-1.fc40.x86_64.rpm +4fdafe5a28dc18a892713cc2071a46cbcb6561c9c62e10f20f04b0e562187228 libidn2-2.3.8-1.fc40.i686.rpm +63a08c0cf18474582a3e62367b5b4275d079e883e40f4cf32cab7afc316ec2dc libidn2-2.3.8-1.fc40.x86_64.rpm +98b0d9d25bd93c7061ce50480e214944a02d7de725e1d31f4461604380ffb74a libkcapi-1.5.0-4.fc40.x86_64.rpm +84977f5f157172dc7642a3f6602692bb6323b4b106c69f7081882e6c6a81a346 libkcapi-hasher-1.5.0-4.fc40.x86_64.rpm +906bb224af7b2e1ea64c258c6978a610b899b0af5be572ce1c09e36ec58b8a79 libkcapi-hmaccalc-1.5.0-4.fc40.x86_64.rpm +a77eed0fe1b84c11f9175f4642db058753d4eaa1f88e999f01df72e1d10a3826 libksba-1.6.6-1.fc40.x86_64.rpm +d5d9c95e38aeb8c852cda4516057d86a5fec2485cb3413067d625059a4d97b30 libmd-1.1.0-4.fc40.x86_64.rpm +7b307e95fb7584889d35108de86ebfa34d0aea6eabb5a68d574647f83f25ed77 libmnl-1.0.5-5.fc40.x86_64.rpm +4aa44fc80c1d3e3496a406740b9ae3ada3df28b37fecc611e44183f542758f76 libmount-2.40-0.9.rc1.fc40.i686.rpm +c5231753426984926daba70cc61d20e777048e91167f2e3c217e21e9205573a6 libmount-2.40-0.9.rc1.fc40.x86_64.rpm +76ae4cbab05343ba545c2c3898153493f0663a2f6ff382df464cebe9d260bab4 libmount-2.40.2-1.fc40.i686.rpm +a695daa293bb78b033a2629f5af1284fe212b748227e94efa59a8292eb6b9f40 libmount-2.40.2-1.fc40.x86_64.rpm +157c1256a9529dea2215f6c77f40647baf19c6f8ac6058934c0f2a593f436c4d libnet-1.3-3.fc40.x86_64.rpm +a2d50812dec895ba654fc424a458e99ceb81423046ad870ffabfef3081382ef6 libnetfilter_conntrack-1.0.9-5.fc40.x86_64.rpm +3de45cea1d877e58f1a8fadb3902e585c070bfd2813bb107145a2f28a2e6edb3 libnetfilter_cthelper-1.0.0-27.fc40.x86_64.rpm +aa2f6d8990f059ba985681d8a6d1695730bbe798014d5bd4b6aa112f79b386b3 libnetfilter_cttimeout-1.0.0-25.fc40.x86_64.rpm +ad8d041ab07f62567f80e9a751529091f591a542dd91b6473b7cba5749b56d69 libnetfilter_queue-1.0.5-8.fc40.x86_64.rpm +78055d1a143d118b9b5513e6621c31d19858c593427b6343a42655eb147a44ed libnfnetlink-1.0.1-27.fc40.x86_64.rpm +db4841a294c5ec3759d77f356a05d0c7f852270aa75c900daac4992f12147dd3 libnftnl-1.2.6-5.fc40.x86_64.rpm +0e9b7c72112f58e83d66422eb2d77d346dc0810cdb652906f0d0fcbd9799fc7d libnghttp2-1.59.0-3.fc40.i686.rpm +550160732fc268914a422cfddc3c745febf8da161f8eacbce8649c67117b1476 libnghttp2-1.59.0-3.fc40.x86_64.rpm +d7062104274c9b8eae85b7c199a69c2f8692c17f31d3fdf4364b53f6a3553e9c libnl3-3.11.0-1.fc40.x86_64.rpm +fa6dccd7aee4a74a5cfa12c7927c7326485704ebe57c54774b0f157fda639360 libnsl2-2.0.1-1.fc40.x86_64.rpm +9e27ce1072ef67dd8877175f9a7daa1bcddbbcec3fd6f161e6bc2f2b453c360b libnvme-1.8-1.fc40.x86_64.rpm +bb9ceaba0d3283777777524e8c99b8eaa2155e9000d8e3ef5d0ece336f8c1392 libpsl-0.21.5-3.fc40.x86_64.rpm +87e8725c378e16a983abee0b8bfbdaf2214f32b55c822741e627db34427ed9a3 libpwquality-1.4.5-9.fc40.i686.rpm +210e797a265da7111c1a59eca95f9e301ad05c5c8772aed54af9363e5684950b libpwquality-1.4.5-9.fc40.x86_64.rpm +571fad7baa286ca36a2b2cdb171d22142ba82b99663ec0408b5db99514773956 libseccomp-2.5.5-1.fc40.i686.rpm +91668f5d08a663948c7d888d7cdef3248285c5d9fbe369ae031d7ca31c6e398c libseccomp-2.5.5-1.fc40.x86_64.rpm +e4d4e12303eeee24bd19b7c1010abf5a275577f5c6aa59ccbd15887e0f5f09ee libsecret-0.21.7-2.fc40.x86_64.rpm +69161fabb22dd4c5c8aeab0b6465dafe06117ce5173aaf4dce425a10cb11c434 libselinux-3.7-5.fc40.i686.rpm +2070bdf786c926400739254f08568ccf564ce613ddacacb36b6a9a499345aa5e libselinux-3.7-5.fc40.x86_64.rpm +aca271d814ee3be14c09963985011c201315a186d3e3b634af8d59cd5eb01208 libselinux-utils-3.7-5.fc40.x86_64.rpm +e200b862d5063f6e85859c5be99c50d5636edae91bd3f603c3a22383b7e2ac88 libsemanage-3.7-2.fc40.x86_64.rpm +a4cd1c54d0f8b543ffa7cc6ce366a6a3f233e084f2e52ea07a70da6127347b8e libsepol-3.7-2.fc40.i686.rpm +85cbaeca877a166cda9637a8ea0d43dd63488fdcc250fe564696cf8beaf8913f libsepol-3.7-2.fc40.x86_64.rpm +716b91d85eb887fe10db607608294475289b9e9fc4d51fbddcf24046ea016147 libsmartcols-2.40-0.9.rc1.fc40.i686.rpm +34111597814e385c8c1cdd48ff72c4ed64e7e6ed9bd6660bb2bfda6aebdb3200 libsmartcols-2.40-0.9.rc1.fc40.x86_64.rpm +e9c3e9e3458af7a2f9b5cd6bc45020bb7f2c6cfbd0429b0b1853928bd3e02004 libsmartcols-2.40.2-1.fc40.x86_64.rpm +45d032fb4d59ee0f6a921dd1f0addfcdd38fc46917243fdd6248194ffddb9067 libsodium-1.0.20-1.fc40.x86_64.rpm +c8bbfa2762cc601f8a97d8d5a39a658f0e91ba477ebebd798b30f7fc8ffdd457 libss-1.47.0-5.fc40.x86_64.rpm +89e7282e0a94d641871dfed423ba2ce6f8b088eaf9aabdea1805708bcafa6a01 libstdc++-14.2.1-3.fc40.x86_64.rpm +3d6ff1e90b4b19de401ab45df9c5bb6e171c34a5b415a7e10e3282332a4cda95 libtasn1-4.20.0-1.fc40.x86_64.rpm +9ca680998686ee852fa8e1667cd6e7c436bfd5fe7da898bd314d808303d447f8 libtextstyle-0.22.5-4.fc40.x86_64.rpm +189e8f25a80a67db1722cb42f2800235df3eadd9cb93d3bcd13853bf09122d5f libtirpc-1.3.6-1.rc3.fc40.x86_64.rpm +e5d150d23f95e4a23288b84145af442607a88bf457c0e04b325b1d1e8e708c2b libtool-ltdl-2.4.7-10.fc40.x86_64.rpm +e541a1c8397dccf159b3602eb6bbb381ba21c544db337a3b3bfc49ccc2ef5c21 libunistring-1.1-7.fc40.i686.rpm +58719c2f205b23598e31b72144ab55215947ad8fca96af46a641288692c159d2 libunistring-1.1-7.fc40.x86_64.rpm +0fa1b7d1f6f5bcd1c2f2785e6571a7c3e63662efd50ba32fd1996dac9dbb4de9 libusb1-1.0.28-2.fc40.x86_64.rpm +896d671852ed3f28e8c778dca361c5b5c57a89855df11755e6be1d088f64d43a libutempter-1.2.1-13.fc40.i686.rpm +0093a8d3f490fbbbc71b01e0c8f9b083040dbf7513be31a91a0769d846198c1b libutempter-1.2.1-13.fc40.x86_64.rpm +5aaa12bba361ae29b2a6b35c4b21da935423bc2ad763eaa8267008c7a533cb3c libuuid-2.40-0.9.rc1.fc40.i686.rpm +18ae5558dd719fcc92bd4d2c7c73c6a093af82a35c67444ccc9cdf4b3dce1824 libuuid-2.40-0.9.rc1.fc40.x86_64.rpm +7273be2566ee7c865ca3154715e2f7fa938e852bc6114af9cb8530fa88d833ca libuuid-2.40.2-1.fc40.i686.rpm +b6db3e72ae6575127216145c1f65414ea94acd9db26d08c5081cb5d786101c1f libuuid-2.40.2-1.fc40.x86_64.rpm +bea578631618692ba5e302beadfdf6d5894e23e5bddaea4b4fca2f377dd1aaac libverto-0.3.2-8.fc40.i686.rpm +fadf7dd93c5eee57ba78e0628bf041dbd2ea037ace52f0a5cbac55b363234d27 libverto-0.3.2-8.fc40.x86_64.rpm +c87a32fd07fd5be227320177ddf61b89c9f14f06d0895e9ecb9a9977b8f6495b libxcrypt-4.4.38-7.fc40.i686.rpm +27f89188ec9f4b1e1e96275f7d2760c342b4cf0a28e7cbccd893d98418d1d060 libxcrypt-4.4.38-7.fc40.x86_64.rpm +a17f9a8894a00ee97a42219b3b21d64bfb850d74059d89ae299210bc477e8967 libxkbcommon-1.6.0-2.fc40.i686.rpm +1f1d0c1e1132016735acc6fc3390102b35f9eb257244547c7b61c32a9c2314cc libxkbcommon-1.6.0-2.fc40.x86_64.rpm +12fa7bdef4a5d95b78a38152a0c90b42c0cfc1a1b7c80fa25f4ccdb7c13cf849 libxml2-2.12.10-1.fc40.i686.rpm +a8ee5e5e972ac86d383bf2798db45f41a22b23d76a0fdef698ddd92076589ff5 libxml2-2.12.10-1.fc40.x86_64.rpm +9007aa6bc776262992172e3f24ea2528cd4f65fb82f2c2d01e27f53f79c5c6f7 libzstd-1.5.7-1.fc40.i686.rpm +ec5650e3822d102bfe6bbebc7468a711128ef695f4bd06748ce242b8378d8b7b libzstd-1.5.7-1.fc40.x86_64.rpm +81409455da42a5ffdcf5b8cc711632ce037fec25d5ae00cbfda5010c9db04157 lua-libs-5.4.6-5.fc40.x86_64.rpm +2d1da8faf26c647a7299f840cfa199f20415ceb99a4f694ac3cd07f645f02cf5 lz4-libs-1.9.4-6.fc40.i686.rpm +f5f022440c4340b5e7fb1c1dbc382e6b0fd57030b3ff056940f2bb3d254408ec lz4-libs-1.9.4-6.fc40.x86_64.rpm +55d5c4384bdc13290a7824aa566d1d20bbbc99f5dde4e057bd12f4f47845e28b memstrack-0.2.5-4.fc40.x86_64.rpm +2030e8622b6f9ceb4b56f5c771ae7e4ccbff3d8bf563df5bf929725f4c4f18c9 mkpasswd-5.5.20-3.fc40.x86_64.rpm +03fbefea8c8d8465cf1caf66870fb935292ee18b4ca341853b5576ca9c7801eb mokutil-0.7.1-1.fc40.x86_64.rpm +0a3a3fc2471d2d64cbc85f4b23c93620df6eeee814851a2b69fc5ddf75406b56 mpdecimal-2.5.1-9.fc40.x86_64.rpm +bc873693a8b8423d7f82e329abe207c9160a4c746fea9a32ef2a6ae8c912f227 mpfr-4.2.1-4.fc40.x86_64.rpm +8a7312e49b3ddec619dee7d1067b72f9105f34d9ff988be0e8b8a76091a8b8fa mtools-4.0.48-1.fc40.x86_64.rpm +7dfae7d898dfc40f3fe1fc66104cf31e434e866fec4d4944b55952d7f2f16657 nano-7.2-7.fc40.x86_64.rpm +b404c27af03bb1e43fb0dc472d5a1fa152e0563fa2e4eefa29199c47578a829b nano-default-editor-7.2-7.fc40.noarch.rpm +8a93376ce7423bd1a649a13f4b5105f270b4603f5cf3b3e230bdbda7f25dd788 ncurses-base-6.4-12.20240127.fc40.noarch.rpm +39bba59320e6276a3b7b07bc94d319511bdd7d32ba098fd49723f4d542794d41 ncurses-libs-6.4-12.20240127.fc40.i686.rpm +a18edf32e89aefd453998d5d0ec3aa1ea193dac43f80b99db195abd7e8cf1a04 ncurses-libs-6.4-12.20240127.fc40.x86_64.rpm +60d1e0058d38ab2ea6b08f59341e7db34c8bec37a387ad5c0565bbc38d5170fd netavark-1.14.1-1.fc40.x86_64.rpm +16172412cfd45453292e18f84fc57e42a3ce92aca72b47ef7e15b44554049cfe nettle-3.9.1-6.fc40.x86_64.rpm +188ce5004e6ed764b4a619b64a4a0f36f1cc4fa919fe0a300599ff1171844144 nftables-1.0.9-3.fc40.x86_64.rpm +784e0fbc9ccb7087c10f4c41edbed13904f94244ff658f308614abe48cdf0d42 npth-1.7-1.fc40.x86_64.rpm +f814bc09b50daaab468715088ec056373dbc209a5075306e4ce76f5c55eb2b42 nvme-cli-2.8-1.fc40.x86_64.rpm +a0eecb082db491d57bfab6047b2611a10150d47a8f50fd05f98ad2f01b0dee54 openldap-2.6.9-1.fc40.x86_64.rpm +49e3e1c7d82ab28b1ab79bb2655a95cadeae2295f5543db8a07d74e090bcb90d openssh-9.6p1-2.fc40.x86_64.rpm +369b4d6e159a53afe9d2803d927f0523ceeae0822353aa8f0d81d0e3211b0788 openssh-server-9.6p1-2.fc40.x86_64.rpm +bffa85f8feadf0bf5f7a8cea9ff9f5e49266959df6ae4d61cf929054c09ec2f8 openssl-libs-3.2.4-1.fc40.i686.rpm +a1b67803e7afb5e16d977e49b8e63c50537bbaa6b261ab10348d55a54b1562dd openssl-libs-3.2.4-1.fc40.x86_64.rpm +9f0336deb6f1b1524ec48d837622e7e2291995369b0356d7ad1e1d427f3b659a os-prober-1.81-6.fc40.x86_64.rpm +70fba929aab38a9d69a457cef1b01962161a1df2b78dc5a4e86ff4b994b51079 p11-kit-0.25.5-1.fc40.x86_64.rpm +c728dbd90872b7597a8ace70a70555bff576231bb6dbde14b75626d601706af8 p11-kit-trust-0.25.5-1.fc40.x86_64.rpm +b3b261e448a25c6550f050ca1813509dd6edbb10f22c02a535548332435b6bc4 pam-1.6.1-5.fc40.x86_64.rpm +753d7b5a6531eec7689414dc1a4ce76ba4d327b8ad0363a9298ee67b565c1d95 pam-libs-1.6.1-5.fc40.i686.rpm +6ca8efd0b2a26cc51917c1c81260d919ef7760f0e0770dc872a78b1b829299cd pam-libs-1.6.1-5.fc40.x86_64.rpm +9bbce784622e02af0371ced8e9a7d26adba7eabd66ecfcb8bbe2d24cf616e3c1 parted-3.6-4.fc40.x86_64.rpm +fa10fa559403d57df8c8dabd9cfd765f020216ca03d2116c861aa7cf7a97b27a passt-0^20250415.g2340bbf-1.fc40.x86_64.rpm +b8892365092573b21fc84ebd084b20f6f62e848ac19720ea8a9e0c2fd64176c4 passt-selinux-0^20250415.g2340bbf-1.fc40.noarch.rpm +a0fb808d6b7ff8cd9cfdc1a60f213851cecdcace334d6e5aa1e0e54b81d79a25 pcre2-10.44-1.fc40.i686.rpm +73e50df09266fcffda9c24a3738f579dd365c2c187c294da054ef9915edc3851 pcre2-10.44-1.fc40.x86_64.rpm +dbec699e88d42fc6fb1df0a8c0b9023941ed1b1b7625694253a612eaf9f2691d pcre2-syntax-10.44-1.fc40.noarch.rpm +d207e7cdb8602403c8aab36c1342f55bcb4503bf4e296d11dae013e6fd9ac920 pcsc-lite-2.0.3-1.fc40.x86_64.rpm +cbd3f6cdbb19126dc703f140fafcac84d0d2ef63b54dfa08332d4bce2def076f pcsc-lite-ccid-1.5.5-3.fc40.x86_64.rpm +f796a31cad58f4ebea8787020868581d9a721297ee0ef6a7c63a7f8444f60c17 pcsc-lite-libs-2.0.3-1.fc40.x86_64.rpm +5443db8875acc0c1c436dbe1ed62b776543e049b8d9c7e33198379d367814093 pigz-2.8-4.fc40.x86_64.rpm +cb7c5036f1d25c696de23a6670cb64caec9945116fb0c9a93555414746ecf253 pinentry-1.3.0-2.fc40.x86_64.rpm +bbb4abafa9f7664e21350b56d49af2c928288e6d4dd68c304c4ab5d45b2c8ad7 pkcs11-provider-0.3-2.fc40.x86_64.rpm +ecdf5f33e98a3c94426efc2176d382899d08d632e05dafc2e7fd18549337ef75 podman-5.4.2-1.fc40.x86_64.rpm +f6291fc1fd3ececcd23c9e693ae0d309d66d57cc2de5d3d389235604804c1c2a policycoreutils-3.7-7.fc40.x86_64.rpm +30a4f9d3631aaa1280c93ce4305847a9773973aa312e1802d1cd676cb2421689 polkit-124-2.fc40.x86_64.rpm +f47bc65177a8b160916c00df9c84442afa1dd353880b3c0503d5a0b052d4956c polkit-libs-124-2.fc40.x86_64.rpm +b7decdd8a6fcb175fea2bb39bb1dbecad1ba820c365bab5a273a7b3982e55157 polkit-pkla-compat-0.1-28.fc40.x86_64.rpm +c03ba1c46e0e2dda36e654941f307aaa0d6574ee5143d6fec6e9af2bdf3252a2 popt-1.19-6.fc40.x86_64.rpm +8a414572157d7e450eddcdc909521e09373289cc7a48ebc15f7b0c9922c17262 procps-ng-4.0.4-3.fc40.x86_64.rpm +af85755cda79959a19161ebc26a45e507003298bd97b472b9ab0d512afa5e46a protobuf-c-1.5.0-3.fc40.x86_64.rpm +45ff2e9814aa059f323b23710c73309d41d36306667a3004f5fbb86b0cab4484 psmisc-23.6-6.fc40.x86_64.rpm +c000cbb0a7df2c0c61559ab3f3732eacd163b171673298f4ec043cb6d223f364 publicsuffix-list-dafsa-20250116-1.fc40.noarch.rpm +7c703b431508f44c5184b5c1df052ed0f49b7439d68aa3597a9a57a5b26bd648 python-pip-wheel-23.3.2-2.fc40.noarch.rpm +bc9b15b36777510ccc9ddc3da363a3100990a235e998f3a50743cdfa50e92f5e python-unversioned-command-3.12.10-2.fc40.noarch.rpm +b0eced6eca5856ed3ddd031e8010e91975149cecf4b337fd3a8c82759a2344c9 python3-3.12.10-2.fc40.x86_64.rpm +c59ab2ff672f5e2d1be59dd22ebbe3fc86c88aca95247b1f84c9bf3d5b6c7026 python3-libs-3.12.10-2.fc40.x86_64.rpm +b593f10e736995a0a898d37a1e595ad04bf162e7f0e7c9994e3032a9d9bc8799 qemu-user-static-8.2.9-1.fc40.x86_64.rpm +72b6185e59a7b359df273e9fe27dc014c0856341abcd1c907c7810a77cc6980e qemu-user-static-aarch64-8.2.9-1.fc40.x86_64.rpm +7eace5bc3e601266452b8356207604012542a8cd019b576e6d2ab9306c6a8e4c qemu-user-static-alpha-8.2.9-1.fc40.x86_64.rpm +6baadf76d6111fabe0c40d9ef8956396ffd23175dadceff940356e1d447c9f0b qemu-user-static-arm-8.2.9-1.fc40.x86_64.rpm +284850c3af8faa4b7ae51b5d2c1c597e783945a20b950b7bc0a925d02538ced3 qemu-user-static-cris-8.2.9-1.fc40.x86_64.rpm +8b5d75d2839682691d130409ebf7142f3fafdb970348f3b5336876d38d4ab0b7 qemu-user-static-hexagon-8.2.9-1.fc40.x86_64.rpm +fef5e7fef2df6f85092383de73373e5d9c9eb241232281bb94c068564c7306de qemu-user-static-hppa-8.2.9-1.fc40.x86_64.rpm +742c694b0e8cbe8bafe2b0602bb38f17c31b80c04f7eb6ee7f104e97d60a1e48 qemu-user-static-loongarch64-8.2.9-1.fc40.x86_64.rpm +872255f64d77ccc3a5388ace420e1434c504e79e31442f58a1516ff752697641 qemu-user-static-m68k-8.2.9-1.fc40.x86_64.rpm +314453265e4fe55fd7b02e01d17c14c4f68ff23bf652ca251915d65ff9006cc4 qemu-user-static-microblaze-8.2.9-1.fc40.x86_64.rpm +decbbd27625256fe9930c908eaf18e6c852f15e47913b7ca071bea091737d5c8 qemu-user-static-mips-8.2.9-1.fc40.x86_64.rpm +f7d088b6b21089c66c24317de994bc65c5ed0f7be484518e72aefe0f74aaf959 qemu-user-static-nios2-8.2.9-1.fc40.x86_64.rpm +58129a9a88a231446ec2fba60c378438d0020e3c7d72186f40d65d06be12eb93 qemu-user-static-or1k-8.2.9-1.fc40.x86_64.rpm +e1c1dc5e5b1d39b52b57c2ebfbbbaa4ece91d986ee8065547bcfb2f6b4707b92 qemu-user-static-ppc-8.2.9-1.fc40.x86_64.rpm +8863c1d75d185ed71bcba9496d353467d064669a3825ab5fda7b4f964bd1a76f qemu-user-static-riscv-8.2.9-1.fc40.x86_64.rpm +d8e5cbc0354aef6bca8600aeeb5f64b4833bb3f90466e64c37695fd7822c5f4b qemu-user-static-s390x-8.2.9-1.fc40.x86_64.rpm +71663f59440aae47599d857f89fb2149952ec4f1bb1afbd83f7d6ea00ccc393f qemu-user-static-sh4-8.2.9-1.fc40.x86_64.rpm +bfef80139dc556e1834b424a346a69fbdc2c706e0158402d8ea6c42e04123bb3 qemu-user-static-sparc-8.2.9-1.fc40.x86_64.rpm +9a0f90c77539963e1657148b3fda7b91285abfbfba52893735b36971df338b6c qemu-user-static-x86-8.2.9-1.fc40.x86_64.rpm +ccf7dd55fc0101233b3384077b544b8fcf658b3febd0e062a687a7537dd9618c qemu-user-static-xtensa-8.2.9-1.fc40.x86_64.rpm +8d50fba416f81e4091b144748fff22665ee88699fdc4a372b905d999d05fd3e8 qrencode-libs-4.1.1-7.fc40.i686.rpm +93781052576cc40a2c203bbc1bf865189a11b2c82436e614da9811baedc082fc qrencode-libs-4.1.1-7.fc40.x86_64.rpm +3527582fddcb54892228658b3929ffbb89766941a9794e726216e0800ac05721 readline-8.2-8.fc40.i686.rpm +dacd59edbe4744fd9f6823d672e01eff89f871e88537554f16c0a275a17d04e9 readline-8.2-8.fc40.x86_64.rpm +2fbe0a8f9925ba12b4307fbed8c5c148bab91835f1a3e8797ee08d94d2a0bf83 rpm-4.19.1.1-1.fc40.x86_64.rpm +c48c149f4aebfe44d649eea6f7a8eaa229dc8db71ff70b66c7403aa9bd072820 rpm-libs-4.19.1.1-1.fc40.x86_64.rpm +7bebda41ea91faf8cf8911a403c051eb59d444e60f8091d14d10987b713f39ff rpm-plugin-audit-4.19.1.1-1.fc40.x86_64.rpm +d400a4e4440bea56566fb1e9582d86d1ac2e07745d37fa6e71f43a8fea05217c rpm-plugin-selinux-4.19.1.1-1.fc40.x86_64.rpm +ce3b3148bb617e132c2ae9a28cc9f1990f806bc45722489f4c09f4d90821b6cd rpm-sequoia-1.7.0-5.fc40.x86_64.rpm +216aedc28a4144469041eade68f57149e2a7ab91c5f4f46eba18b6fc6effcb73 runc-1.3.0-1.fc40.x86_64.rpm +5dbd069183076ed8048c839c31f713c0f6080fb9ebfdda92ac550030688e811b sbsigntools-0.9.5-6.fc40.x86_64.rpm +6a21b2c132a54fd6d9acb846d0a96289ab739b745cdc4c2b31bdbf6b2434a1a7 sed-4.9-1.fc40.x86_64.rpm +4ea87bc61621f2465a6db2bf14144acd37230132fc84a2d3af485067646e37e1 selinux-policy-40.30-1.fc40.noarch.rpm +428f1d37fc81965af5962d65b7232b6425db2ff3b3778475756e690acb04a51d selinux-policy-targeted-40.30-1.fc40.noarch.rpm +89862f646cd64e81497f01a8b69ab30ac8968c47afef92a2c333608fdb90ccc1 setup-2.14.5-2.fc40.noarch.rpm +cfde0d25ecac7e689ee083b330b78df51d346c2b7557c83a189d5df95c4e2c8d shadow-utils-4.15.1-4.fc40.x86_64.rpm +6e9b6b6196f1782419e447ac806c762d002c6930fe39b18999d9b32c24a0ecfc shadow-utils-subid-4.15.1-4.fc40.x86_64.rpm +67eede27af5b4773eb2f7ac794df694be030310d40bce462864c05b8f65c87c3 socat-1.8.0.0-2.fc40.x86_64.rpm +9fe46c08d942a5eaa66d997368f372557a81383fe9831ddeb801bccdde64f28b sqlite-libs-3.45.1-3.fc40.x86_64.rpm +08c40fb895e75add08d2c239d7bd24a0633ffaafed430f7ad5b464d1eea2a8b6 systemd-255.18-1.fc40.i686.rpm +51cd1eaa48613b981940d81ae76ea610c28265104e289a434a845ebf9e8f85a4 systemd-255.18-1.fc40.x86_64.rpm +d3c0e2fcfce8b412667f58d63f927076b01f51e0f36f5a7d4b4038efbebc95a0 systemd-boot-unsigned-255.18-1.fc40.x86_64.rpm +1fa7a7e4fcfa4f78c9bb0b5f95e100d3c9f36841368189339449612e7edb2a3c systemd-libs-255.18-1.fc40.i686.rpm +aceabc4c1848f8154afba961b3dfac1c95ec6c72cdf9238ceca384b699af3cef systemd-libs-255.18-1.fc40.x86_64.rpm +c8f5b5c50202965f86dc9139de3082a972e2a0222a1a426ce76a30d47cba5f32 systemd-networkd-255.18-1.fc40.x86_64.rpm +746e0db539b7c9a8bba425957304ca311878b198a2f6d63e60c42c266688c1dc systemd-pam-255.18-1.fc40.i686.rpm +71e42c637702d75b5a66058cb087c4ef7d0475da635ccfd0f40f71372d0e7de7 systemd-pam-255.18-1.fc40.x86_64.rpm +07082ff756a9db4851f996eb03e75f2d499a26bd42211a984788af5a1eeed300 systemd-resolved-255.18-1.fc40.x86_64.rpm +5dc7bdcbb590edf79d8ae90bf99a94d535b055816864ffdb2546acafc97d2736 systemd-udev-255.18-1.fc40.x86_64.rpm +65819c502727dc293a71a74b9a5f6b0ba781f12a99c5d5535085f168e5eac56e tar-1.35-3.fc40.x86_64.rpm +0478e12152cc3432a31dfca5ddbc80966800af437c6d7c0b26be307d5e1272e7 tpm2-tools-5.7-1.fc40.x86_64.rpm +c3be8a6d0ea23b1d0bf466b19857b97f7ffde811ad7adec0599161059d84cc74 tpm2-tss-4.1.3-1.fc40.x86_64.rpm +5df98756883badf7743cdd75f5689b62606bff0b74494b20241cb9d78335c251 tpm2-tss-fapi-4.1.3-1.fc40.x86_64.rpm +d35ca6852dfea66d10046dd8b38a77e89443ce2006bc31782abfead826dba029 tzdata-2025b-1.fc40.noarch.rpm +e1d443f7dcaec55eedc34bb66dd798ba9901dba69a169cff46f6c45671a3b3fa unbound-anchor-1.21.1-11.fc40.x86_64.rpm +8eb278cecd9f28fa4131dc402a31c74c427626aae53b2231bb452e745a9e9346 unbound-libs-1.21.1-11.fc40.x86_64.rpm +36ffa617a0dfe523424a28290241a81cd51f7d82e776e58131f16d092d49797b util-linux-2.40-0.9.rc1.fc40.i686.rpm +945aa536bc30050abc1870cef167cb944cf78d6628923476db43201a0054574b util-linux-2.40.2-1.fc40.x86_64.rpm +7ec1b5df780c5a30f8e901179480125a6ea87f1f7bad3b69da7f4b351b88c3dd util-linux-core-2.40-0.9.rc1.fc40.x86_64.rpm +b1aa4e816c01c08c18924865640f214f717cdfc66837e53a24b8edfb80a86f9d util-linux-core-2.40.2-1.fc40.x86_64.rpm +673532a506dff0ca46cd4bb5fbf772d2039e4c11e648eafa221f207139ebb798 vim-common-9.1.1275-1.fc40.x86_64.rpm +d32e2c404e54d75fb7c7c4cd3dece12123418f1798a964047fe5ca70221db002 vim-data-9.1.1275-1.fc40.noarch.rpm +968283f6290df2cbf2dd699411b1cca450769b30f28b8b8a9a9f4a4916d5ae4f vim-enhanced-9.1.1275-1.fc40.x86_64.rpm +80194d554770c211bd7c3fa368b3fbb94f7021504d4c87ddae4544a6eb183342 vim-filesystem-9.1.1275-1.fc40.noarch.rpm +69fd53fe41a8811e904b5429b1934a413d88978ec54d5c9e64370be32cbfc2ef wget2-2.2.0-2.fc40.x86_64.rpm +a00cc0a87c60ffbf5495a9796ac7074e6a47e0bebbb8c137d902014cd7ff5a30 wget2-libs-2.2.0-2.fc40.x86_64.rpm +09822d8d386dc81619639415a211b34592b0c5d43f7be288691cc4d933a0542c wget2-wget-2.2.0-2.fc40.x86_64.rpm +cf0306ceed1c6b3be39060d85f16b1953b464d3a625488b170d3b7aadf600645 which-2.21-41.fc40.x86_64.rpm +4ede95a2fa3bc0ae617c8bf3a375b800163d58733b4829b15d9f038505d79fee whois-nls-5.5.20-3.fc40.noarch.rpm +e2195010e857f56b19246f8b821f9391922880b7691b3728a413f540edc890a6 xkeyboard-config-2.41-1.fc40.noarch.rpm +69b64249d3c26a5efbd0ffa5802aa08033822c2d4378a4f44261618de6d38e4d xxd-9.1.1275-1.fc40.x86_64.rpm +9a0f6eb8d2784d7e3ee062c5deb3b9af41e7e6c6d115b6da8420bde453e41744 xz-5.8.1-2.fc40.x86_64.rpm +c6d64a788bfbbedc6eeab2347274864cade272c4ad69d77cb30d8b602e25e7f0 xz-libs-5.8.1-2.fc40.i686.rpm +cbad4c25b1acbf152273c2fae29e42ddffa03414938a7c755b5afa5f2ba45a26 xz-libs-5.8.1-2.fc40.x86_64.rpm +9e263e0a9b656178519de20733f3e0950fef494aa056daaa2004b522ba50b952 yajl-2.1.0-23.fc40.x86_64.rpm +ffab1c8720480b498f65d0d480825ccd890e4f797c3850712879eb04a4739690 zlib-ng-compat-2.1.7-2.fc40.i686.rpm +e50b69054de16d757f5667e3acf2e7439302c91a9c418243467f288dfb79f6ea zlib-ng-compat-2.1.7-2.fc40.x86_64.rpm diff --git a/image/mirror/dnf.conf b/image/mirror/dnf.conf index c298b8587..32618492e 100644 --- a/image/mirror/dnf.conf +++ b/image/mirror/dnf.conf @@ -8,4 +8,4 @@ best=False skip_if_unavailable=True tsflags=nodocs basearch=x86_64 -releasever=38 +releasever=40 diff --git a/image/mirror/packages.txt b/image/mirror/packages.txt index 10768f830..9d6240a2e 100644 --- a/image/mirror/packages.txt +++ b/image/mirror/packages.txt @@ -19,6 +19,8 @@ mokutil nano nano-default-editor nvme-cli +openssh-server +passt-selinux passwd podman sbsigntools diff --git a/image/mirror/update_packages.sh b/image/mirror/update_packages.sh index f2245d262..b177f6b4a 100755 --- a/image/mirror/update_packages.sh +++ b/image/mirror/update_packages.sh @@ -39,7 +39,7 @@ download() { "${DNF5}" \ "--config=${DNF_CONF}" \ "--setopt=reposdir=${REPOSDIR}" \ - "--releasever=38" \ + "--releasever=40" \ download \ "--destdir=${OUTDIR}" \ --resolve --alldeps \ diff --git a/image/sysroot-tree/etc/ssh/sshd_config b/image/sysroot-tree/etc/ssh/sshd_config new file mode 100644 index 000000000..39016f323 --- /dev/null +++ b/image/sysroot-tree/etc/ssh/sshd_config @@ -0,0 +1,5 @@ +HostKey /var/run/state/ssh/ssh_host_ed25519_key +HostCertificate /var/run/state/ssh/ssh_host_cert.pub +TrustedUserCAKeys /var/run/state/ssh/ssh_ca.pub +PasswordAuthentication no +ChallengeResponseAuthentication no diff --git a/image/sysroot-tree/etc/systemd/system/sshd-keygen@.service.d/override.conf b/image/sysroot-tree/etc/systemd/system/sshd-keygen@.service.d/override.conf new file mode 100644 index 000000000..1e956c08b --- /dev/null +++ b/image/sysroot-tree/etc/systemd/system/sshd-keygen@.service.d/override.conf @@ -0,0 +1,3 @@ +[Unit] +ConditionFileNotEmpty=|!/var/run/state/ssh/ssh_host_%i_key +Before=constellation-bootstrapper.service diff --git a/image/sysroot-tree/usr/lib/systemd/system/getty@tty1.service.d/autologin.conf b/image/sysroot-tree/usr/lib/systemd/system/getty@tty1.service.d/autologin.conf new file mode 100644 index 000000000..ec52d1369 --- /dev/null +++ b/image/sysroot-tree/usr/lib/systemd/system/getty@tty1.service.d/autologin.conf @@ -0,0 +1,11 @@ +[Unit] +Description=autologin +ConditionPathExists=/proc/cmdline +ConditionKernelCommandLine=|constel.console +ConditionKernelCommandLine=|constel.debug + +[Service] +ExecStart= +ExecStart=-/sbin/agetty -o '-p -f -- \\u' --noclear --autologin root %I $TERM +[Install] +WantedBy=multi-user.target diff --git a/image/sysroot-tree/usr/lib/systemd/system/serial-getty@ttyS0.service.d/autologin.conf b/image/sysroot-tree/usr/lib/systemd/system/serial-getty@ttyS0.service.d/autologin.conf new file mode 100644 index 000000000..24fe28a99 --- /dev/null +++ b/image/sysroot-tree/usr/lib/systemd/system/serial-getty@ttyS0.service.d/autologin.conf @@ -0,0 +1,12 @@ +[Unit] +Description=autologin +ConditionPathExists=/proc/cmdline +ConditionKernelCommandLine=|constel.console +ConditionKernelCommandLine=|constel.debug + +[Service] +ExecStart= +ExecStart=-/sbin/agetty -o '-p -f -- \\u' --keep-baud --autologin root 115200,57600,38400,9600 - $TERM + +[Install] +WantedBy=multi-user.target diff --git a/image/sysroot-tree/usr/lib/systemd/system/sshd-keygen.target b/image/sysroot-tree/usr/lib/systemd/system/sshd-keygen.target new file mode 100644 index 000000000..3c4dd2b1c --- /dev/null +++ b/image/sysroot-tree/usr/lib/systemd/system/sshd-keygen.target @@ -0,0 +1,3 @@ +[Unit] +Wants=sshd-keygen@ed25519.service +PartOf=sshd.service diff --git a/image/sysroot-tree/usr/libexec/openssh/sshd-keygen b/image/sysroot-tree/usr/libexec/openssh/sshd-keygen new file mode 100644 index 000000000..c366b0d0a --- /dev/null +++ b/image/sysroot-tree/usr/libexec/openssh/sshd-keygen @@ -0,0 +1,44 @@ +#!/usr/bin/bash +# Taken from the original openssh-server package and slightly modified + +set -x + +# Create the host keys for the OpenSSH server. +KEYTYPE=$1 +case $KEYTYPE in +"dsa") ;& # disabled in FIPS +"ed25519") + FIPS=/proc/sys/crypto/fips_enabled + if [[ -r $FIPS && $(cat $FIPS) == "1" ]]; then + exit 0 + fi + ;; +"rsa") ;; # always ok +"ecdsa") ;; +*) # wrong argument + exit 12 ;; +esac +mkdir -p /var/run/state/ssh +KEY=/var/run/state/ssh/ssh_host_${KEYTYPE}_key + +KEYGEN=/usr/bin/ssh-keygen +if [[ ! -x $KEYGEN ]]; then + exit 13 +fi + +# remove old keys +rm -f "$KEY"{,.pub} + +# create new keys +if ! $KEYGEN -q -t "$KEYTYPE" -f "$KEY" -C '' -N '' >&/dev/null; then + exit 1 +fi + +# sanitize permissions +/usr/bin/chmod 600 "$KEY" +/usr/bin/chmod 644 "$KEY".pub +if [[ -x /usr/sbin/restorecon ]]; then + /usr/sbin/restorecon "$KEY"{,.pub} +fi + +exit 0 diff --git a/image/system/BUILD.bazel b/image/system/BUILD.bazel index ddc7ae621..3a51c92a4 100644 --- a/image/system/BUILD.bazel +++ b/image/system/BUILD.bazel @@ -1,5 +1,6 @@ load("//bazel/mkosi:mkosi_image.bzl", "mkosi_image") -load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "autologin", "constellation_packages", "images_for_csp", "images_for_csp_and_stream", "images_for_stream", "kernel_command_line", "kernel_command_line_dict") +load("//bazel/osimage:upload_os_images.bzl", "upload_os_images") +load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "base_image", "constellation_packages", "images_for_csp", "images_for_csp_and_stream", "images_for_stream", "kernel_command_line", "kernel_command_line_dict") [ mkosi_image( @@ -9,13 +10,12 @@ load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "autologin", "constellation ] + glob([ "mkosi.repart/**", ]), - autologin = autologin( - variant["csp"], - variant["attestation_variant"], - stream, - ), base_trees = [ - "//image/base:image.tar", + base_image( + variant["csp"], + variant["attestation_variant"], + stream, + ), ], extra_trees = constellation_packages(stream), initrds = [ @@ -38,6 +38,19 @@ load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "autologin", "constellation "no-cache", ], version_file = "//bazel/settings:tag", + visibility = ["//visibility:public"], + ) + for variant in VARIANTS + for stream in STREAMS +] + +[ + upload_os_images( + name = "upload_" + variant["csp"] + "_" + variant["attestation_variant"] + "_" + stream, + image_dirs = [":" + variant["csp"] + "_" + variant["attestation_variant"] + "_" + stream], + tags = [ + "manual", + ], ) for variant in VARIANTS for stream in STREAMS @@ -51,6 +64,18 @@ load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "autologin", "constellation "manual", "no-cache", ], + visibility = ["//visibility:public"], + ) + for stream in STREAMS +] + +[ + upload_os_images( + name = "upload_" + stream, + image_dirs = [":" + stream], + tags = [ + "manual", + ], ) for stream in STREAMS ] @@ -63,6 +88,7 @@ load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "autologin", "constellation "manual", "no-cache", ], + visibility = ["//visibility:public"], ) for csp in CSPS ] @@ -75,6 +101,7 @@ load(":variants.bzl", "CSPS", "STREAMS", "VARIANTS", "autologin", "constellation "manual", "no-cache", ], + visibility = ["//visibility:public"], ) for csp in CSPS for stream in STREAMS diff --git a/image/system/mkosi.conf b/image/system/mkosi.conf index f49c9ebd8..d97bbc1bb 100644 --- a/image/system/mkosi.conf +++ b/image/system/mkosi.conf @@ -1,6 +1,7 @@ [Distribution] Distribution=fedora -Release=38 +Release=40 +RepositoryKeyFetch=yes [Output] Format=disk @@ -13,7 +14,7 @@ Seed=0e9a6fe0-68f6-408c-bbeb-136054d20445 SourceDateEpoch=0 Bootable=yes Bootloader=uki -KernelCommandLine=preempt=full rd.shell=0 rd.emergency=reboot loglevel=8 console=ttyS0 +KernelCommandLine=preempt=full rd.shell=0 rd.emergency=reboot loglevel=8 RemoveFiles=/var/log RemoveFiles=/var/cache RemoveFiles=/etc/pki/ca-trust/extracted/java/cacerts diff --git a/image/system/variants.bzl b/image/system/variants.bzl index 9f96ba7a8..b9b1c6bd8 100644 --- a/image/system/variants.bzl +++ b/image/system/variants.bzl @@ -13,6 +13,10 @@ VARIANTS = [ "attestation_variant": "azure-sev-snp", "csp": "azure", }, + { + "attestation_variant": "azure-tdx", + "csp": "azure", + }, { "attestation_variant": "gcp-sev-es", "csp": "gcp", @@ -46,14 +50,14 @@ CSPS = [ "qemu", ] -base_cmdline = "selinux=1 enforcing=0 audit=0" +base_cmdline = "selinux=1 enforcing=0 audit=0 console=tty1 console=ttyS0" csp_settings = { "aws": { "kernel_command_line_dict": { + "console": "ttyS0", "constel.csp": "aws", - "idle": "poll", - "mitigations": "auto", + "mitigations": "auto,nosmt", }, }, "azure": { @@ -69,10 +73,7 @@ csp_settings = { }, }, "openstack": { - "autologin": True, - "kernel_command_line": "console=tty0 console=ttyS0", "kernel_command_line_dict": { - "console": "tty0", "constel.csp": "openstack", "kvm_amd.sev": "1", "mem_encrypt": "on", @@ -81,7 +82,7 @@ csp_settings = { }, }, "qemu": { - "autologin": True, + "kernel_command_line": "constel.console", # All qemu images have console enabled independent of stream "kernel_command_line_dict": { "constel.csp": "qemu", "mitigations": "auto,nosmt", @@ -105,6 +106,12 @@ attestation_variant_settings = { "constel.attestation-variant": "azure-sev-snp", }, }, + "azure-tdx": { + "base_image": "//image/base:mainline", + "kernel_command_line_dict": { + "constel.attestation-variant": "azure-tdx", + }, + }, "gcp-sev-es": { "kernel_command_line_dict": { "constel.attestation-variant": "gcp-sev-es", @@ -124,11 +131,10 @@ attestation_variant_settings = { stream_settings = { "console": { - "autologin": True, + "kernel_command_line": "constel.console", }, "debug": { - "autologin": True, - "kernel_command_line": "constellation.debug", + "kernel_command_line": "constel.debug", }, "nightly": {}, "stable": {}, @@ -169,26 +175,6 @@ def constellation_packages(stream): "//bootstrapper/cmd/bootstrapper:bootstrapper-package", ] + base_packages -def autologin(csp, attestation_variant, stream): - """Generates a boolean indicating whether autologin should be enabled for the given csp, attestation_variant and stream. - - Args: - csp: The cloud service provider to use. - attestation_variant: The attestation variant to use. - stream: The stream to use. - - Returns: - A boolean indicating whether autologin should be enabled. - """ - out = None - for settings in from_settings(csp, attestation_variant, stream): - if not "autologin" in settings: - continue - if out != None and out != settings["autologin"]: - fail("Inconsistent autologin settings") - out = settings["autologin"] - return out - def kernel_command_line(csp, attestation_variant, stream): cmdline = base_cmdline for settings in from_settings(csp, attestation_variant, stream, default = {}): @@ -201,6 +187,12 @@ def kernel_command_line_dict(csp, attestation_variant, stream): commandline_dict = commandline_dict | settings.get("kernel_command_line_dict", {}) return commandline_dict +def base_image(csp, attestation_variant, stream): + for settings in from_settings(csp, attestation_variant, stream): + if "base_image" in settings: + return settings["base_image"] + return "//image/base:lts" + def append_cmdline(current, append): """Append a string to an existing commandline, separating them with a space. diff --git a/image/upload/internal/cmd/BUILD.bazel b/image/upload/internal/cmd/BUILD.bazel index e89acc665..ac78c4c2e 100644 --- a/image/upload/internal/cmd/BUILD.bazel +++ b/image/upload/internal/cmd/BUILD.bazel @@ -4,22 +4,15 @@ go_library( name = "cmd", srcs = [ "api.go", - "aws.go", - "azure.go", "flags.go", - "gcp.go", - "image.go", "info.go", "measurements.go", "measurementsenvelope.go", "measurementsmerge.go", "measurementsupload.go", "must.go", - "nop.go", - "openstack.go", - "qemu.go", - "secureboot.go", "upload.go", + "uplosi.go", ], importpath = "github.com/edgelesssys/constellation/v2/image/upload/internal/cmd", visibility = ["//image/upload:__subpackages__"], @@ -27,18 +20,14 @@ go_library( "//internal/api/versionsapi", "//internal/attestation/measurements", "//internal/cloud/cloudprovider", + "//internal/constants", "//internal/logger", "//internal/osimage", "//internal/osimage/archive", - "//internal/osimage/aws", - "//internal/osimage/azure", - "//internal/osimage/gcp", "//internal/osimage/imageinfo", "//internal/osimage/measurementsuploader", "//internal/osimage/nop", - "//internal/osimage/secureboot", - "@com_github_spf13_afero//:afero", + "//internal/osimage/uplosi", "@com_github_spf13_cobra//:cobra", - "@org_uber_go_zap//zapcore", ], ) diff --git a/image/upload/internal/cmd/api.go b/image/upload/internal/cmd/api.go index 5f6865998..5cf4e0b95 100644 --- a/image/upload/internal/cmd/api.go +++ b/image/upload/internal/cmd/api.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/image/upload/internal/cmd/aws.go b/image/upload/internal/cmd/aws.go deleted file mode 100644 index b2c4f0058..000000000 --- a/image/upload/internal/cmd/aws.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "fmt" - "io" - "os" - - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/osimage" - "github.com/edgelesssys/constellation/v2/internal/osimage/archive" - awsupload "github.com/edgelesssys/constellation/v2/internal/osimage/aws" - "github.com/spf13/cobra" -) - -// newAWSCmd returns the command that uploads an OS image to AWS. -func newAWSCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "aws", - Short: "Upload OS image to AWS", - Long: "Upload OS image to AWS.", - Args: cobra.ExactArgs(0), - RunE: runAWS, - } - - cmd.Flags().String("aws-region", "eu-central-1", "AWS region used during AMI creation") - cmd.Flags().String("aws-bucket", "constellation-images", "S3 bucket used during AMI creation") - return cmd -} - -func runAWS(cmd *cobra.Command, _ []string) error { - workdir := os.Getenv("BUILD_WORKING_DIRECTORY") - if len(workdir) > 0 { - must(os.Chdir(workdir)) - } - - flags, err := parseAWSFlags(cmd) - if err != nil { - return err - } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) - - archiveC, archiveCClose, err := archive.New(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) - if err != nil { - return err - } - defer func() { - if err := archiveCClose(cmd.Context()); err != nil { - log.Errorf("closing archive client: %v", err) - } - }() - - uploadC, err := awsupload.New(flags.awsRegion, flags.awsBucket, log) - if err != nil { - return fmt.Errorf("uploading image: %w", err) - } - - file, err := os.Open(flags.rawImage) - if err != nil { - return fmt.Errorf("uploading image: opening image file %w", err) - } - defer file.Close() - size, err := file.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - out := cmd.OutOrStdout() - if len(flags.out) > 0 { - outF, err := os.Create(flags.out) - if err != nil { - return fmt.Errorf("uploading image: opening output file %w", err) - } - defer outF.Close() - out = outF - } - - uploadReq := &osimage.UploadRequest{ - Provider: flags.provider, - Version: flags.version, - AttestationVariant: flags.attestationVariant, - SecureBoot: flags.secureBoot, - Size: size, - Timestamp: flags.timestamp, - Image: file, - } - - if flags.secureBoot { - sbDatabase, uefiVarStore, err := loadSecureBootKeys(flags.pki) - if err != nil { - return err - } - uploadReq.SBDatabase = sbDatabase - uploadReq.UEFIVarStore = uefiVarStore - } - - return uploadImage(cmd.Context(), archiveC, uploadC, uploadReq, out) -} diff --git a/image/upload/internal/cmd/azure.go b/image/upload/internal/cmd/azure.go deleted file mode 100644 index 18491bd07..000000000 --- a/image/upload/internal/cmd/azure.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "fmt" - "io" - "os" - - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/osimage" - "github.com/edgelesssys/constellation/v2/internal/osimage/archive" - azureupload "github.com/edgelesssys/constellation/v2/internal/osimage/azure" - "github.com/spf13/cobra" -) - -// newAzureCmd returns the command that uploads an OS image to Azure. -func newAzureCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "azure", - Short: "Upload OS image to Azure", - Long: "Upload OS image to Azure.", - Args: cobra.ExactArgs(0), - RunE: runAzure, - } - - cmd.Flags().String("az-subscription", "0d202bbb-4fa7-4af8-8125-58c269a05435", "Azure subscription to use") - cmd.Flags().String("az-location", "northeurope", "Azure location to use") - cmd.Flags().String("az-resource-group", "constellation-images", "Azure resource group to use") - return cmd -} - -func runAzure(cmd *cobra.Command, _ []string) error { - workdir := os.Getenv("BUILD_WORKING_DIRECTORY") - if len(workdir) > 0 { - must(os.Chdir(workdir)) - } - - flags, err := parseAzureFlags(cmd) - if err != nil { - return err - } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) - - archiveC, archiveCClose, err := archive.New(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) - if err != nil { - return err - } - defer func() { - if err := archiveCClose(cmd.Context()); err != nil { - log.Errorf("closing archive client: %v", err) - } - }() - - uploadC, err := azureupload.New(flags.azSubscription, flags.azLocation, flags.azResourceGroup, log) - if err != nil { - return fmt.Errorf("uploading image: %w", err) - } - - file, err := os.Open(flags.rawImage) - if err != nil { - return fmt.Errorf("uploading image: opening image file %w", err) - } - defer file.Close() - size, err := file.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - out := cmd.OutOrStdout() - if len(flags.out) > 0 { - outF, err := os.Create(flags.out) - if err != nil { - return fmt.Errorf("uploading image: opening output file %w", err) - } - defer outF.Close() - out = outF - } - - uploadReq := &osimage.UploadRequest{ - Provider: flags.provider, - Version: flags.version, - AttestationVariant: flags.attestationVariant, - SecureBoot: flags.secureBoot, - Size: size, - Timestamp: flags.timestamp, - Image: file, - } - - if flags.secureBoot { - sbDatabase, uefiVarStore, err := loadSecureBootKeys(flags.pki) - if err != nil { - return err - } - uploadReq.SBDatabase = sbDatabase - uploadReq.UEFIVarStore = uefiVarStore - } - - return uploadImage(cmd.Context(), archiveC, uploadC, uploadReq, out) -} diff --git a/image/upload/internal/cmd/flags.go b/image/upload/internal/cmd/flags.go index b0cf85f6d..de38c2701 100644 --- a/image/upload/internal/cmd/flags.go +++ b/image/upload/internal/cmd/flags.go @@ -1,222 +1,27 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd import ( "errors" - "os" + "log/slog" "path/filepath" - "time" + "strings" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) -type commonFlags struct { - rawImage string - pki string - provider cloudprovider.Provider - attestationVariant string - secureBoot bool - version versionsapi.Version - timestamp time.Time - region string - bucket string - distributionID string - out string - logLevel zapcore.Level -} - -func parseCommonFlags(cmd *cobra.Command) (commonFlags, error) { - workspaceDir := os.Getenv("BUILD_WORKSPACE_DIRECTORY") - rawImage, err := cmd.Flags().GetString("raw-image") - if err != nil { - return commonFlags{}, err - } - pki, err := cmd.Flags().GetString("pki") - if err != nil { - return commonFlags{}, err - } - if pki == "" { - pki = filepath.Join(workspaceDir, "image/pki") - } - attestationVariant, err := cmd.Flags().GetString("attestation-variant") - if err != nil { - return commonFlags{}, err - } - secureBoot, err := cmd.Flags().GetBool("secure-boot") - if err != nil { - return commonFlags{}, err - } - version, err := cmd.Flags().GetString("version") - if err != nil { - return commonFlags{}, err - } - ver, err := versionsapi.NewVersionFromShortPath(version, versionsapi.VersionKindImage) - if err != nil { - return commonFlags{}, err - } - timestamp, err := cmd.Flags().GetString("timestamp") - if err != nil { - return commonFlags{}, err - } - if timestamp == "" { - timestamp = time.Now().Format("2006-01-02T15:04:05Z07:00") - } - timestmp, err := time.Parse("2006-01-02T15:04:05Z07:00", timestamp) - if err != nil { - return commonFlags{}, err - } - region, err := cmd.Flags().GetString("region") - if err != nil { - return commonFlags{}, err - } - bucket, err := cmd.Flags().GetString("bucket") - if err != nil { - return commonFlags{}, err - } - distributionID, err := cmd.Flags().GetString("distribution-id") - if err != nil { - return commonFlags{}, err - } - out, err := cmd.Flags().GetString("out") - if err != nil { - return commonFlags{}, err - } - verbose, err := cmd.Flags().GetBool("verbose") - if err != nil { - return commonFlags{}, err - } - logLevel := zapcore.InfoLevel - if verbose { - logLevel = zapcore.DebugLevel - } - - return commonFlags{ - rawImage: rawImage, - pki: pki, - attestationVariant: attestationVariant, - secureBoot: secureBoot, - version: ver, - timestamp: timestmp, - region: region, - bucket: bucket, - distributionID: distributionID, - out: out, - logLevel: logLevel, - }, nil -} - -type awsFlags struct { - commonFlags - awsRegion string - awsBucket string -} - -func parseAWSFlags(cmd *cobra.Command) (awsFlags, error) { - common, err := parseCommonFlags(cmd) - if err != nil { - return awsFlags{}, err - } - - awsRegion, err := cmd.Flags().GetString("aws-region") - if err != nil { - return awsFlags{}, err - } - awsBucket, err := cmd.Flags().GetString("aws-bucket") - if err != nil { - return awsFlags{}, err - } - - common.provider = cloudprovider.AWS - return awsFlags{ - commonFlags: common, - awsRegion: awsRegion, - awsBucket: awsBucket, - }, nil -} - -type azureFlags struct { - commonFlags - azSubscription string - azLocation string - azResourceGroup string -} - -func parseAzureFlags(cmd *cobra.Command) (azureFlags, error) { - common, err := parseCommonFlags(cmd) - if err != nil { - return azureFlags{}, err - } - - azSubscription, err := cmd.Flags().GetString("az-subscription") - if err != nil { - return azureFlags{}, err - } - azLocation, err := cmd.Flags().GetString("az-location") - if err != nil { - return azureFlags{}, err - } - azResourceGroup, err := cmd.Flags().GetString("az-resource-group") - if err != nil { - return azureFlags{}, err - } - - common.provider = cloudprovider.Azure - return azureFlags{ - commonFlags: common, - azSubscription: azSubscription, - azLocation: azLocation, - azResourceGroup: azResourceGroup, - }, nil -} - -type gcpFlags struct { - commonFlags - gcpProject string - gcpLocation string - gcpBucket string -} - -func parseGCPFlags(cmd *cobra.Command) (gcpFlags, error) { - common, err := parseCommonFlags(cmd) - if err != nil { - return gcpFlags{}, err - } - - gcpProject, err := cmd.Flags().GetString("gcp-project") - if err != nil { - return gcpFlags{}, err - } - gcpLocation, err := cmd.Flags().GetString("gcp-location") - if err != nil { - return gcpFlags{}, err - } - gcpBucket, err := cmd.Flags().GetString("gcp-bucket") - if err != nil { - return gcpFlags{}, err - } - - common.provider = cloudprovider.GCP - return gcpFlags{ - commonFlags: common, - gcpProject: gcpProject, - gcpLocation: gcpLocation, - gcpBucket: gcpBucket, - }, nil -} - type s3Flags struct { region string bucket string distributionID string - logLevel zapcore.Level + logLevel slog.Level } func parseS3Flags(cmd *cobra.Command) (s3Flags, error) { @@ -236,9 +41,9 @@ func parseS3Flags(cmd *cobra.Command) (s3Flags, error) { if err != nil { return s3Flags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return s3Flags{ @@ -279,7 +84,7 @@ func parseUploadMeasurementsFlags(cmd *cobra.Command) (measurementsFlags, error) type mergeMeasurementsFlags struct { out string - logLevel zapcore.Level + logLevel slog.Level } func parseMergeMeasurementsFlags(cmd *cobra.Command) (mergeMeasurementsFlags, error) { @@ -291,9 +96,9 @@ func parseMergeMeasurementsFlags(cmd *cobra.Command) (mergeMeasurementsFlags, er if err != nil { return mergeMeasurementsFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return mergeMeasurementsFlags{ @@ -307,7 +112,7 @@ type envelopeMeasurementsFlags struct { csp cloudprovider.Provider attestationVariant string in, out string - logLevel zapcore.Level + logLevel slog.Level } func parseEnvelopeMeasurementsFlags(cmd *cobra.Command) (envelopeMeasurementsFlags, error) { @@ -343,9 +148,9 @@ func parseEnvelopeMeasurementsFlags(cmd *cobra.Command) (envelopeMeasurementsFla if err != nil { return envelopeMeasurementsFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return envelopeMeasurementsFlags{ @@ -357,3 +162,130 @@ func parseEnvelopeMeasurementsFlags(cmd *cobra.Command) (envelopeMeasurementsFla logLevel: logLevel, }, nil } + +type uplosiFlags struct { + rawImage string + provider cloudprovider.Provider + version versionsapi.Version + attestationVariant string + out string + uplosiPath string + + // archiver flags + region string + bucket string + distributionID string + + logLevel slog.Level +} + +func parseUplosiFlags(cmd *cobra.Command) (uplosiFlags, error) { + rawImage, err := cmd.Flags().GetString("raw-image") + if err != nil { + return uplosiFlags{}, err + } + rawImage, err = filepath.Abs(rawImage) + if err != nil { + return uplosiFlags{}, err + } + + // extract csp, attestation variant, and stream from the raw image name + // format: __/constellation.raw + + var guessedCSP, guessedAttestationVariant, guessedStream string + pathComponents := strings.Split(filepath.ToSlash(rawImage), "/") + if len(pathComponents) >= 2 && pathComponents[len(pathComponents)-1] == "constellation.raw" { + imageMetadata := pathComponents[len(pathComponents)-2] + imageMetadataComponents := strings.Split(imageMetadata, "_") + if len(imageMetadataComponents) == 3 { + guessedCSP = imageMetadataComponents[0] + guessedAttestationVariant = imageMetadataComponents[1] + guessedStream = imageMetadataComponents[2] + } + } + + csp, err := cmd.Flags().GetString("csp") + if err != nil || csp == "" { + csp = guessedCSP + } + if csp == "" { + if err != nil { + return uplosiFlags{}, err + } + return uplosiFlags{}, errors.New("csp is required") + } + provider := cloudprovider.FromString(csp) + ref, err := cmd.Flags().GetString("ref") + if err != nil { + return uplosiFlags{}, err + } + stream, err := cmd.Flags().GetString("stream") + if err != nil || stream == "" { + stream = guessedStream + } + if stream == "" { + if err != nil { + return uplosiFlags{}, err + } + return uplosiFlags{}, errors.New("stream is required") + } + version, err := cmd.Flags().GetString("version") + if err != nil { + return uplosiFlags{}, err + } + ver, err := versionsapi.NewVersion(ref, stream, version, versionsapi.VersionKindImage) + if err != nil { + return uplosiFlags{}, err + } + attestationVariant, err := cmd.Flags().GetString("attestation-variant") + if err != nil || attestationVariant == "" { + attestationVariant = guessedAttestationVariant + } + if attestationVariant == "" { + if err != nil { + return uplosiFlags{}, err + } + return uplosiFlags{}, errors.New("attestation-variant is required") + } + out, err := cmd.Flags().GetString("out") + if err != nil { + return uplosiFlags{}, err + } + uplosiPath, err := cmd.Flags().GetString("uplosi-path") + if err != nil { + return uplosiFlags{}, err + } + region, err := cmd.Flags().GetString("region") + if err != nil { + return uplosiFlags{}, err + } + bucket, err := cmd.Flags().GetString("bucket") + if err != nil { + return uplosiFlags{}, err + } + distributionID, err := cmd.Flags().GetString("distribution-id") + if err != nil { + return uplosiFlags{}, err + } + verbose, err := cmd.Flags().GetBool("verbose") + if err != nil { + return uplosiFlags{}, err + } + logLevel := slog.LevelInfo + if verbose { + logLevel = slog.LevelDebug + } + + return uplosiFlags{ + rawImage: rawImage, + provider: provider, + version: ver, + attestationVariant: attestationVariant, + out: out, + uplosiPath: uplosiPath, + region: region, + bucket: bucket, + distributionID: distributionID, + logLevel: logLevel, + }, nil +} diff --git a/image/upload/internal/cmd/gcp.go b/image/upload/internal/cmd/gcp.go deleted file mode 100644 index 2cf1bc7e8..000000000 --- a/image/upload/internal/cmd/gcp.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "fmt" - "io" - "os" - - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/osimage" - "github.com/edgelesssys/constellation/v2/internal/osimage/archive" - gcpupload "github.com/edgelesssys/constellation/v2/internal/osimage/gcp" - "github.com/spf13/cobra" -) - -// newGCPCommand returns the command that uploads an OS image to GCP. -func newGCPCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "gcp", - Short: "Upload OS image to GCP", - Long: "Upload OS image to GCP.", - Args: cobra.ExactArgs(0), - RunE: runGCP, - } - - cmd.Flags().String("gcp-project", "constellation-images", "GCP project to use") - cmd.Flags().String("gcp-location", "europe-west3", "GCP location to use") - cmd.Flags().String("gcp-bucket", "constellation-os-images", "GCP bucket to use") - return cmd -} - -func runGCP(cmd *cobra.Command, _ []string) error { - workdir := os.Getenv("BUILD_WORKING_DIRECTORY") - if len(workdir) > 0 { - must(os.Chdir(workdir)) - } - - flags, err := parseGCPFlags(cmd) - if err != nil { - return err - } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) - - archiveC, archiveCClose, err := archive.New(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) - if err != nil { - return err - } - defer func() { - if err := archiveCClose(cmd.Context()); err != nil { - log.Errorf("closing archive client: %v", err) - } - }() - - uploadC, err := gcpupload.New(cmd.Context(), flags.gcpProject, flags.gcpLocation, flags.gcpBucket, log) - if err != nil { - return fmt.Errorf("uploading image: %w", err) - } - - file, err := os.Open(flags.rawImage) - if err != nil { - return fmt.Errorf("uploading image: opening image file %w", err) - } - defer file.Close() - size, err := file.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - out := cmd.OutOrStdout() - if len(flags.out) > 0 { - outF, err := os.Create(flags.out) - if err != nil { - return fmt.Errorf("uploading image: opening output file %w", err) - } - defer outF.Close() - out = outF - } - - uploadReq := &osimage.UploadRequest{ - Provider: flags.provider, - Version: flags.version, - AttestationVariant: flags.attestationVariant, - SecureBoot: flags.secureBoot, - Size: size, - Timestamp: flags.timestamp, - Image: file, - } - - if flags.secureBoot { - sbDatabase, uefiVarStore, err := loadSecureBootKeys(flags.pki) - if err != nil { - return err - } - uploadReq.SBDatabase = sbDatabase - uploadReq.UEFIVarStore = uefiVarStore - } - - return uploadImage(cmd.Context(), archiveC, uploadC, uploadReq, out) -} diff --git a/image/upload/internal/cmd/image.go b/image/upload/internal/cmd/image.go deleted file mode 100644 index 0f4e94e4c..000000000 --- a/image/upload/internal/cmd/image.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "os" - - "github.com/spf13/cobra" -) - -// NewImageCmd creates a new image parent command. Image needs another -// verb, and does nothing on its own. -func NewImageCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "image", - Short: "Uploads OS images to supported CSPs", - Long: "Uploads OS images to supported CSPs.", - Args: cobra.ExactArgs(0), - } - - cmd.SetOut(os.Stdout) - - cmd.PersistentFlags().String("raw-image", "", "Path to os image in CSP specific format that should be uploaded.") - cmd.PersistentFlags().Bool("secure-boot", false, "Enables secure boot support.") - cmd.PersistentFlags().String("pki", "", "Base path to the PKI (secure boot signing) files.") - cmd.PersistentFlags().String("attestation-variant", "", "Attestation variant of the image being uploaded.") - cmd.PersistentFlags().String("version", "", "Shortname of the os image version.") - cmd.PersistentFlags().String("timestamp", "", "Optional timestamp to use for resource names. Uses format 2006-01-02T15:04:05Z07:00.") - cmd.PersistentFlags().String("region", "eu-central-1", "AWS region of the archive S3 bucket") - cmd.PersistentFlags().String("bucket", "cdn-constellation-backend", "S3 bucket name of the archive") - cmd.PersistentFlags().String("distribution-id", "E1H77EZTHC3NE4", "CloudFront distribution ID of the API") - cmd.PersistentFlags().String("out", "", "Optional path to write the upload result to. If not set, the result is written to stdout.") - cmd.PersistentFlags().Bool("verbose", false, "Enable verbose output") - must(cmd.MarkPersistentFlagRequired("raw-image")) - must(cmd.MarkPersistentFlagRequired("attestation-variant")) - must(cmd.MarkPersistentFlagRequired("version")) - - cmd.AddCommand(newAWSCmd()) - cmd.AddCommand(newAzureCmd()) - cmd.AddCommand(newGCPCommand()) - cmd.AddCommand(newOpenStackCmd()) - cmd.AddCommand(newQEMUCmd()) - - return cmd -} diff --git a/image/upload/internal/cmd/info.go b/image/upload/internal/cmd/info.go index 51ba533cc..d104d1dc2 100644 --- a/image/upload/internal/cmd/info.go +++ b/image/upload/internal/cmd/info.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -12,6 +12,7 @@ import ( "os" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/logger" infoupload "github.com/edgelesssys/constellation/v2/internal/osimage/imageinfo" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ func NewInfoCmd() *cobra.Command { cmd.Flags().String("region", "eu-central-1", "AWS region of the archive S3 bucket") cmd.Flags().String("bucket", "cdn-constellation-backend", "S3 bucket name of the archive") - cmd.Flags().String("distribution-id", "E1H77EZTHC3NE4", "CloudFront distribution ID of the API") + cmd.Flags().String("distribution-id", constants.CDNDefaultDistributionID, "CloudFront distribution ID of the API") cmd.Flags().Bool("verbose", false, "Enable verbose output") return cmd @@ -48,8 +49,8 @@ func runInfo(cmd *cobra.Command, args []string) error { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "region", flags.region, "bucket", flags.bucket, "distributionID", flags.distributionID) info, err := readInfoArgs(args) if err != nil { return err @@ -61,7 +62,7 @@ func runInfo(cmd *cobra.Command, args []string) error { } defer func() { if err := uploadCClose(cmd.Context()); err != nil { - log.Errorf("closing upload client: %v", err) + log.Error(fmt.Sprintf("closing upload client: %v", err)) } }() @@ -69,7 +70,7 @@ func runInfo(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("uploading image info: %w", err) } - log.Infof("Uploaded image info to %s", url) + log.Info(fmt.Sprintf("Uploaded image info to %s", url)) return nil } diff --git a/image/upload/internal/cmd/measurements.go b/image/upload/internal/cmd/measurements.go index e117b88d9..a61c14c0d 100644 --- a/image/upload/internal/cmd/measurements.go +++ b/image/upload/internal/cmd/measurements.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/image/upload/internal/cmd/measurementsenvelope.go b/image/upload/internal/cmd/measurementsenvelope.go index 1f6c49b02..082e82ea7 100644 --- a/image/upload/internal/cmd/measurementsenvelope.go +++ b/image/upload/internal/cmd/measurementsenvelope.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -53,8 +53,8 @@ func runEnvelopeMeasurements(cmd *cobra.Command, _ []string) error { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "version", flags.version.Version(), "csp", flags.csp, "attestationVariant", flags.attestationVariant, "in", flags.in) f, err := os.Open(flags.in) if err != nil { @@ -66,6 +66,11 @@ func runEnvelopeMeasurements(cmd *cobra.Command, _ []string) error { return fmt.Errorf("enveloping measurements: reading input file: %w", err) } + measuremnt.Measurements, err = measurements.ApplyOverrides(measuremnt.Measurements, flags.csp, flags.attestationVariant) + if err != nil { + return fmt.Errorf("enveloping measurements: overriding static measurements: %w", err) + } + enveloped := measurements.ImageMeasurementsV2{ Ref: flags.version.Ref(), Stream: flags.version.Stream(), @@ -92,7 +97,7 @@ func runEnvelopeMeasurements(cmd *cobra.Command, _ []string) error { if err := json.NewEncoder(out).Encode(enveloped); err != nil { return fmt.Errorf("enveloping measurements: writing output file: %w", err) } - log.Infof("Enveloped image measurements") + log.Info("Enveloped image measurements") return nil } diff --git a/image/upload/internal/cmd/measurementsmerge.go b/image/upload/internal/cmd/measurementsmerge.go index 758f54e5d..4a74e20b2 100644 --- a/image/upload/internal/cmd/measurementsmerge.go +++ b/image/upload/internal/cmd/measurementsmerge.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -44,8 +44,8 @@ func runMergeMeasurements(cmd *cobra.Command, args []string) error { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "out", flags.out, "logLevel", flags.logLevel) mergedMeasurements, err := readMeasurementsArgs(args) if err != nil { @@ -65,7 +65,7 @@ func runMergeMeasurements(cmd *cobra.Command, args []string) error { if err := json.NewEncoder(out).Encode(mergedMeasurements); err != nil { return fmt.Errorf("merging measurements: writing output file: %w", err) } - log.Infof("Merged image measurements") + log.Info("Merged image measurements") return nil } diff --git a/image/upload/internal/cmd/measurementsupload.go b/image/upload/internal/cmd/measurementsupload.go index 4398a0dfc..bca15e121 100644 --- a/image/upload/internal/cmd/measurementsupload.go +++ b/image/upload/internal/cmd/measurementsupload.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -10,6 +10,7 @@ import ( "fmt" "os" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/osimage/measurementsuploader" "github.com/spf13/cobra" @@ -31,7 +32,7 @@ func newMeasurementsUploadCmd() *cobra.Command { cmd.Flags().String("signature", "", "Path to signature file to upload") cmd.Flags().String("region", "eu-central-1", "AWS region of the archive S3 bucket") cmd.Flags().String("bucket", "cdn-constellation-backend", "S3 bucket name of the archive") - cmd.Flags().String("distribution-id", "E1H77EZTHC3NE4", "CloudFront distribution ID of the API") + cmd.Flags().String("distribution-id", constants.CDNDefaultDistributionID, "CloudFront distribution ID of the API") cmd.Flags().Bool("verbose", false, "Enable verbose output") must(cmd.MarkFlagRequired("measurements")) @@ -51,8 +52,8 @@ func runMeasurementsUpload(cmd *cobra.Command, _ []string) error { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "measurementsPath", flags.measurementsPath, "signaturePath", flags.signaturePath, "region", flags.region, "bucket", flags.bucket, "distributionID", flags.distributionID) uploadC, uploadCClose, err := measurementsuploader.New(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) if err != nil { @@ -60,7 +61,7 @@ func runMeasurementsUpload(cmd *cobra.Command, _ []string) error { } defer func() { if err := uploadCClose(cmd.Context()); err != nil { - log.Errorf("closing upload client: %v", err) + log.Error("closing upload client", "error", err) } }() @@ -79,6 +80,6 @@ func runMeasurementsUpload(cmd *cobra.Command, _ []string) error { if err != nil { return fmt.Errorf("uploading image info: %w", err) } - log.Infof("Uploaded image measurements to %s (and signature to %s)", measurementsURL, signatureURL) + log.Info(fmt.Sprintf("Uploaded image measurements to %s (and signature to %s)", measurementsURL, signatureURL)) return nil } diff --git a/image/upload/internal/cmd/must.go b/image/upload/internal/cmd/must.go index fb26f2df0..2f3d8d896 100644 --- a/image/upload/internal/cmd/must.go +++ b/image/upload/internal/cmd/must.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd diff --git a/image/upload/internal/cmd/nop.go b/image/upload/internal/cmd/nop.go deleted file mode 100644 index f6da987ba..000000000 --- a/image/upload/internal/cmd/nop.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "fmt" - "io" - "os" - - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/osimage" - "github.com/edgelesssys/constellation/v2/internal/osimage/archive" - nopupload "github.com/edgelesssys/constellation/v2/internal/osimage/nop" - "github.com/spf13/cobra" -) - -func runNOP(cmd *cobra.Command, provider cloudprovider.Provider, _ []string) error { - workdir := os.Getenv("BUILD_WORKING_DIRECTORY") - if len(workdir) > 0 { - must(os.Chdir(workdir)) - } - - flags, err := parseCommonFlags(cmd) - if err != nil { - return err - } - flags.provider = provider - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) - - archiveC, archiveCClose, err := archive.New(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) - if err != nil { - return err - } - defer func() { - if err := archiveCClose(cmd.Context()); err != nil { - log.Errorf("closing archive client: %v", err) - } - }() - - uploadC := nopupload.New(log) - - file, err := os.Open(flags.rawImage) - if err != nil { - return fmt.Errorf("uploading image: opening image file %w", err) - } - defer file.Close() - size, err := file.Seek(0, io.SeekEnd) - if err != nil { - return err - } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - out := cmd.OutOrStdout() - if len(flags.out) > 0 { - outF, err := os.Create(flags.out) - if err != nil { - return fmt.Errorf("uploading image: opening output file %w", err) - } - defer outF.Close() - out = outF - } - - uploadReq := &osimage.UploadRequest{ - Provider: flags.provider, - Version: flags.version, - AttestationVariant: flags.attestationVariant, - SecureBoot: flags.secureBoot, - Size: size, - Timestamp: flags.timestamp, - Image: file, - } - - if flags.secureBoot { - sbDatabase, uefiVarStore, err := loadSecureBootKeys(flags.pki) - if err != nil { - return err - } - uploadReq.SBDatabase = sbDatabase - uploadReq.UEFIVarStore = uefiVarStore - } - - return uploadImage(cmd.Context(), archiveC, uploadC, uploadReq, out) -} diff --git a/image/upload/internal/cmd/openstack.go b/image/upload/internal/cmd/openstack.go deleted file mode 100644 index b8810c183..000000000 --- a/image/upload/internal/cmd/openstack.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/spf13/cobra" -) - -// newOpenStackCmd returns the command that uploads an OS image to OpenStack. -func newOpenStackCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "openstack", - Short: "Upload OS image to OpenStack", - Long: "Upload OS image to OpenStack.", - Args: cobra.ExactArgs(0), - RunE: runOpenStack, - } - - return cmd -} - -func runOpenStack(cmd *cobra.Command, args []string) error { - return runNOP(cmd, cloudprovider.OpenStack, args) -} diff --git a/image/upload/internal/cmd/qemu.go b/image/upload/internal/cmd/qemu.go deleted file mode 100644 index d34348010..000000000 --- a/image/upload/internal/cmd/qemu.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/spf13/cobra" -) - -// newQEMUCmd returns the command that uploads an OS image to QEMU. -func newQEMUCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "qemu", - Short: "Upload OS image to QEMU", - Long: "Upload OS image to QEMU.", - Args: cobra.ExactArgs(0), - RunE: runQEMU, - } - - return cmd -} - -func runQEMU(cmd *cobra.Command, args []string) error { - return runNOP(cmd, cloudprovider.QEMU, args) -} diff --git a/image/upload/internal/cmd/secureboot.go b/image/upload/internal/cmd/secureboot.go deleted file mode 100644 index a73f2af0f..000000000 --- a/image/upload/internal/cmd/secureboot.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package cmd - -import ( - "fmt" - "path/filepath" - - "github.com/edgelesssys/constellation/v2/internal/osimage/secureboot" - "github.com/spf13/afero" -) - -func loadSecureBootKeys(basePath string) (secureboot.Database, secureboot.UEFIVarStore, error) { - platformKeyCert := filepath.Join(basePath, "PK.cer") - keyExchangeKeyCerts := []string{ - filepath.Join(basePath, "KEK.cer"), - filepath.Join(basePath, "MicCorKEKCA2011_2011-06-24.crt"), - } - signatureDBCerts := []string{ - filepath.Join(basePath, "db.cer"), - filepath.Join(basePath, "MicWinProPCA2011_2011-10-19.crt"), - filepath.Join(basePath, "MicCorUEFCA2011_2011-06-27.crt"), - } - sbDatabase, err := secureboot.DatabaseFromFiles(afero.NewOsFs(), platformKeyCert, keyExchangeKeyCerts, signatureDBCerts) - if err != nil { - return secureboot.Database{}, - secureboot.UEFIVarStore{}, - fmt.Errorf("preparing secure boot database: %w", err) - } - platformKeyESL := filepath.Join(basePath, "PK.esl") - keyExchangeKeyESL := filepath.Join(basePath, "KEK.esl") - signatureDBESL := filepath.Join(basePath, "db.esl") - uefiVarStore, err := secureboot.VarStoreFromFiles(afero.NewOsFs(), platformKeyESL, keyExchangeKeyESL, signatureDBESL, "") - if err != nil { - return secureboot.Database{}, - secureboot.UEFIVarStore{}, - fmt.Errorf("preparing secure boot variable store: %w", err) - } - return sbDatabase, uefiVarStore, nil -} diff --git a/image/upload/internal/cmd/upload.go b/image/upload/internal/cmd/upload.go index 5fdd4bec7..680b26586 100644 --- a/image/upload/internal/cmd/upload.go +++ b/image/upload/internal/cmd/upload.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cmd @@ -14,19 +14,27 @@ import ( "strings" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/osimage" ) func uploadImage(ctx context.Context, archiveC archivist, uploadC uploader, req *osimage.UploadRequest, out io.Writer) error { // upload to S3 archive - archiveURL, err := archiveC.Archive(ctx, req.Version, strings.ToLower(req.Provider.String()), req.AttestationVariant, req.Image) + imageReader, err := req.ImageReader() if err != nil { return err } - // rewind reader so we can read again - if _, err := req.Image.Seek(0, io.SeekStart); err != nil { - return err + defer imageReader.Close() + + var archiveURL string + if needsArchival(req.Provider, req.Version) { + var err error + archiveURL, err = archiveC.Archive(ctx, req.Version, strings.ToLower(req.Provider.String()), req.AttestationVariant, imageReader) + if err != nil { + return err + } } + // upload to CSP imageReferences, err := uploadC.Upload(ctx, req) if err != nil { @@ -55,3 +63,12 @@ func uploadImage(ctx context.Context, archiveC archivist, uploadC uploader, req return nil } + +func needsArchival(provider cloudprovider.Provider, version versionsapi.Version) bool { + switch provider { + case cloudprovider.OpenStack, cloudprovider.QEMU: // image upload for some CSPs only consists of this archival step + return true + } + + return version.Stream() == "stable" || version.Ref() == "-" +} diff --git a/image/upload/internal/cmd/uplosi.go b/image/upload/internal/cmd/uplosi.go new file mode 100644 index 000000000..7e6214837 --- /dev/null +++ b/image/upload/internal/cmd/uplosi.go @@ -0,0 +1,115 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package cmd + +import ( + "fmt" + "io" + "os" + "strconv" + "time" + + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/logger" + "github.com/edgelesssys/constellation/v2/internal/osimage" + "github.com/edgelesssys/constellation/v2/internal/osimage/archive" + nopupload "github.com/edgelesssys/constellation/v2/internal/osimage/nop" + uplosiupload "github.com/edgelesssys/constellation/v2/internal/osimage/uplosi" + "github.com/spf13/cobra" +) + +// NewUplosiCmd returns the command that uses uplosi for uploading os images. +func NewUplosiCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "uplosi", + Short: "Templates uplosi configuration files", + Long: "Templates uplosi configuration files.", + Args: cobra.ExactArgs(0), + RunE: runUplosi, + } + + cmd.SetOut(os.Stdout) + + cmd.Flags().String("raw-image", "", "Path to os image in CSP specific format that should be uploaded.") + cmd.Flags().String("attestation-variant", "", "Attestation variant of the image being uploaded.") + cmd.Flags().String("csp", "", "Cloud service provider that we want to upload this image to. If not set, the csp is guessed from the raw-image file name.") + cmd.Flags().String("ref", "", "Ref of the OS image (part of image shortname).") + cmd.Flags().String("stream", "", "Stream of the OS image (part of the image shortname).") + cmd.Flags().String("version", "", "Semantic version of the os image (part of the image shortname).") + cmd.Flags().String("region", "eu-central-1", "AWS region of the archive S3 bucket") + cmd.Flags().String("bucket", "cdn-constellation-backend", "S3 bucket name of the archive") + cmd.Flags().String("distribution-id", "E1H77EZTHC3NE4", "CloudFront distribution ID of the API") + cmd.Flags().String("out", "", "Optional path to write the upload result to. If not set, the result is written to stdout.") + cmd.Flags().String("uplosi-path", "uplosi", "Path to the uplosi binary.") + cmd.Flags().Bool("verbose", false, "Enable verbose output") + must(cmd.MarkFlagRequired("raw-image")) + must(cmd.MarkFlagRequired("version")) + must(cmd.MarkFlagRequired("ref")) + + return cmd +} + +func runUplosi(cmd *cobra.Command, _ []string) error { + flags, err := parseUplosiFlags(cmd) + if err != nil { + return err + } + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "raw-image", flags.rawImage, "attestation-variant", flags.attestationVariant, "csp", flags.provider, "ref", flags.version.Ref(), "stream", flags.version.Stream(), + "version", flags.version.Version(), "region", flags.region, "bucket", flags.bucket, "distribution-id", flags.distributionID, "out", flags.out, "uplosi-path", flags.uplosiPath) + archiveC, archiveCClose, err := archive.New(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) + if err != nil { + return err + } + defer func() { + if err := archiveCClose(cmd.Context()); err != nil { + log.Error(fmt.Sprintf("closing archive client: %v", err)) + } + }() + + var uploadC uploader + switch flags.provider { + case cloudprovider.AWS, cloudprovider.Azure, cloudprovider.GCP, cloudprovider.OpenStack: + uploadC = uplosiupload.New(flags.uplosiPath, log) + default: + uploadC = nopupload.New(log) + } + + imageOpener := func() (io.ReadSeekCloser, error) { + return os.Open(flags.rawImage) + } + + out := cmd.OutOrStdout() + if len(flags.out) > 0 { + outF, err := os.Create(flags.out) + if err != nil { + return fmt.Errorf("uploading image: opening output file %w", err) + } + defer outF.Close() + out = outF + } + + uploadReq := &osimage.UploadRequest{ + Provider: flags.provider, + Version: flags.version, + AttestationVariant: flags.attestationVariant, + Timestamp: getTimestamp(), + ImageReader: imageOpener, + ImagePath: flags.rawImage, + } + + return uploadImage(cmd.Context(), archiveC, uploadC, uploadReq, out) +} + +func getTimestamp() time.Time { + epoch := os.Getenv("SOURCE_DATE_EPOCH") + epochSecs, err := strconv.ParseInt(epoch, 10, 64) + if epoch == "" || err != nil { + return time.Now().UTC() + } + return time.Unix(epochSecs, 0).UTC() +} diff --git a/image/upload/upload.go b/image/upload/upload.go index a26c79121..2a2ab1423 100644 --- a/image/upload/upload.go +++ b/image/upload/upload.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // upload uploads os images. @@ -40,7 +40,7 @@ func newRootCmd() *cobra.Command { rootCmd.SetOut(os.Stdout) - rootCmd.AddCommand(cmd.NewImageCmd()) + rootCmd.AddCommand(cmd.NewUplosiCmd()) rootCmd.AddCommand(cmd.NewInfoCmd()) rootCmd.AddCommand(cmd.NewMeasurementsCmd()) diff --git a/internal/api/attestationconfigapi/BUILD.bazel b/internal/api/attestationconfigapi/BUILD.bazel index 037f482c9..80397adb3 100644 --- a/internal/api/attestationconfigapi/BUILD.bazel +++ b/internal/api/attestationconfigapi/BUILD.bazel @@ -5,33 +5,24 @@ go_library( name = "attestationconfigapi", srcs = [ "attestationconfigapi.go", - "client.go", "fetcher.go", - "reporter.go", - "snp.go", + "version.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi", visibility = ["//:__subpackages__"], deps = [ - "//internal/api/client", "//internal/api/fetcher", "//internal/attestation/variant", "//internal/constants", - "//internal/logger", "//internal/sigstore", - "//internal/staticupload", - "@com_github_aws_aws_sdk_go//aws", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", ], ) go_test( name = "attestationconfigapi_test", srcs = [ - "client_test.go", "fetcher_test.go", - "reporter_test.go", - "snp_test.go", + "version_test.go", ], embed = [":attestationconfigapi"], deps = [ diff --git a/internal/api/attestationconfigapi/attestationconfigapi.go b/internal/api/attestationconfigapi/attestationconfigapi.go index e170da869..d0ff50872 100644 --- a/internal/api/attestationconfigapi/attestationconfigapi.go +++ b/internal/api/attestationconfigapi/attestationconfigapi.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -15,7 +15,7 @@ information contained in the objects. Especially the paths used for the API are in these helper methods. Regarding the decision to implement new types over using the existing types from internal/config: -AttesationCfg objects for AttestationCfg API need to hold some version information (for sorting, recognizing latest). +AttestationCfg objects for AttestationCfg API need to hold some version information (for sorting, recognizing latest). Thus, existing config types (AWSNitroTPM, AzureSEVSNP, ...) can not be extended to implement apiObject interface. Instead, we need a separate type that wraps _all_ attestation types. In the codebase this is done using the AttestationCfg interface. The new type AttestationCfgGet needs to be located inside internal/config in order to implement UnmarshalJSON. diff --git a/internal/api/attestationconfigapi/cli/BUILD.bazel b/internal/api/attestationconfigapi/cli/BUILD.bazel index c235cfd7c..df2856aeb 100644 --- a/internal/api/attestationconfigapi/cli/BUILD.bazel +++ b/internal/api/attestationconfigapi/cli/BUILD.bazel @@ -10,8 +10,7 @@ go_binary( go_library( name = "cli_lib", srcs = [ - "aws.go", - "azure.go", + "compare.go", "delete.go", "main.go", "upload.go", @@ -21,19 +20,20 @@ go_library( visibility = ["//visibility:private"], deps = [ "//internal/api/attestationconfigapi", + "//internal/api/attestationconfigapi/cli/client", + "//internal/api/fetcher", "//internal/attestation/variant", - "//internal/cloud/cloudprovider", "//internal/constants", "//internal/file", "//internal/logger", "//internal/staticupload", "//internal/verify", - "@com_github_aws_aws_sdk_go//aws", + "@com_github_aws_aws_sdk_go_v2//aws", "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", "@com_github_aws_aws_sdk_go_v2_service_s3//types", + "@com_github_google_go_tdx_guest//proto/tdx", "@com_github_spf13_afero//:afero", "@com_github_spf13_cobra//:cobra", - "@org_uber_go_zap//:zap", ], ) diff --git a/internal/api/attestationconfigapi/cli/aws.go b/internal/api/attestationconfigapi/cli/aws.go deleted file mode 100644 index 578caadc4..000000000 --- a/internal/api/attestationconfigapi/cli/aws.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package main - -import ( - "context" - "fmt" - - "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" -) - -func deleteAWS(ctx context.Context, client *attestationconfigapi.Client, cfg deleteConfig) error { - if cfg.provider != cloudprovider.AWS || cfg.kind != snpReport { - return fmt.Errorf("provider %s and kind %s not supported", cfg.provider, cfg.kind) - } - - return client.DeleteSEVSNPVersion(ctx, variant.AWSSEVSNP{}, cfg.version) -} diff --git a/internal/api/attestationconfigapi/cli/azure.go b/internal/api/attestationconfigapi/cli/azure.go deleted file mode 100644 index 924fbe19e..000000000 --- a/internal/api/attestationconfigapi/cli/azure.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package main - -import ( - "context" - "fmt" - - "github.com/aws/aws-sdk-go-v2/service/s3" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/aws-sdk-go/aws" - "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/staticupload" -) - -func deleteAzure(ctx context.Context, client *attestationconfigapi.Client, cfg deleteConfig) error { - if cfg.provider != cloudprovider.Azure && cfg.kind != snpReport { - return fmt.Errorf("provider %s and kind %s not supported", cfg.provider, cfg.kind) - } - - return client.DeleteSEVSNPVersion(ctx, variant.AzureSEVSNP{}, cfg.version) -} - -func deleteRecursive(ctx context.Context, path string, client *staticupload.Client, cfg deleteConfig) error { - resp, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(cfg.bucket), - Prefix: aws.String(path), - }) - if err != nil { - return err - } - - // Delete all objects in the path. - objIDs := make([]s3types.ObjectIdentifier, len(resp.Contents)) - for i, obj := range resp.Contents { - objIDs[i] = s3types.ObjectIdentifier{Key: obj.Key} - } - if len(objIDs) > 0 { - _, err = client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ - Bucket: aws.String(cfg.bucket), - Delete: &s3types.Delete{ - Objects: objIDs, - Quiet: true, - }, - }) - if err != nil { - return err - } - } - return nil -} diff --git a/internal/api/attestationconfigapi/cli/client/BUILD.bazel b/internal/api/attestationconfigapi/cli/client/BUILD.bazel new file mode 100644 index 000000000..c90cb34b9 --- /dev/null +++ b/internal/api/attestationconfigapi/cli/client/BUILD.bazel @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "client", + srcs = [ + "client.go", + "reporter.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi/cli/client", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/api/attestationconfigapi", + "//internal/api/client", + "//internal/attestation/variant", + "//internal/sigstore", + "//internal/staticupload", + "@com_github_aws_aws_sdk_go//aws", + "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + ], +) + +go_test( + name = "client_test", + srcs = [ + "client_test.go", + "reporter_test.go", + ], + embed = [":client"], + deps = [ + "//internal/api/attestationconfigapi", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/internal/api/attestationconfigapi/cli/client/client.go b/internal/api/attestationconfigapi/cli/client/client.go new file mode 100644 index 000000000..ea867dd1c --- /dev/null +++ b/internal/api/attestationconfigapi/cli/client/client.go @@ -0,0 +1,178 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +/* +package client contains code to manage CVM versions in Constellation's CDN API. +It is used to upload and delete "latest" versions for AMD SEV-SNP and Intel TDX. +*/ +package client + +import ( + "context" + "errors" + "fmt" + "log/slog" + "path" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go/aws" + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + apiclient "github.com/edgelesssys/constellation/v2/internal/api/client" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/sigstore" + + "github.com/edgelesssys/constellation/v2/internal/staticupload" +) + +// VersionFormat is the format of the version name in the S3 bucket. +const VersionFormat = "2006-01-02-15-04" + +// Client manages (modifies) the version information for the attestation variants. +type Client struct { + s3Client *apiclient.Client + s3ClientClose func(ctx context.Context) error + bucketID string + signer sigstore.Signer + cacheWindowSize int + + log *slog.Logger +} + +// New returns a new Client. +func New(ctx context.Context, cfg staticupload.Config, cosignPwd, privateKey []byte, dryRun bool, versionWindowSize int, log *slog.Logger) (*Client, apiclient.CloseFunc, error) { + s3Client, clientClose, err := apiclient.NewClient(ctx, cfg.Region, cfg.Bucket, cfg.DistributionID, dryRun, log) + if err != nil { + return nil, nil, fmt.Errorf("failed to create s3 storage: %w", err) + } + + repo := &Client{ + s3Client: s3Client, + s3ClientClose: clientClose, + signer: sigstore.NewSigner(cosignPwd, privateKey), + bucketID: cfg.Bucket, + cacheWindowSize: versionWindowSize, + log: log, + } + return repo, clientClose, nil +} + +// DeleteVersion deletes the given version (without .json suffix) from the API. +func (c Client) DeleteVersion(ctx context.Context, attestation variant.Variant, versionStr string) error { + versions, err := c.List(ctx, attestation) + if err != nil { + return fmt.Errorf("fetch version list: %w", err) + } + + ops, err := c.deleteVersion(versions, versionStr) + if err != nil { + return err + } + return executeAllCmds(ctx, c.s3Client, ops) +} + +// List returns the list of versions for the given attestation variant. +func (c Client) List(ctx context.Context, attestation variant.Variant) (attestationconfigapi.List, error) { + versions, err := apiclient.Fetch(ctx, c.s3Client, attestationconfigapi.List{Variant: attestation}) + if err != nil { + var notFoundErr *apiclient.NotFoundError + if errors.As(err, ¬FoundErr) { + return attestationconfigapi.List{Variant: attestation}, nil + } + return attestationconfigapi.List{}, err + } + + versions.Variant = attestation + + return versions, nil +} + +func (c Client) deleteVersion(versions attestationconfigapi.List, versionStr string) (ops []crudCmd, err error) { + versionStr = versionStr + ".json" + ops = append(ops, deleteCmd{ + apiObject: attestationconfigapi.Entry{ + Variant: versions.Variant, + Version: versionStr, + }, + }) + + removedVersions, err := removeVersion(versions, versionStr) + if err != nil { + return nil, err + } + ops = append(ops, putCmd{ + apiObject: removedVersions, + signer: c.signer, + }) + return ops, nil +} + +func (c Client) listCachedVersions(ctx context.Context, attestation variant.Variant) ([]string, error) { + list, err := c.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucketID), + Prefix: aws.String(reportVersionDir(attestation)), + }) + if err != nil { + return nil, fmt.Errorf("list objects: %w", err) + } + + var dates []string + for _, obj := range list.Contents { + fileName := path.Base(*obj.Key) + + // The cache contains signature and json files + // We only want the json files + if date, ok := strings.CutSuffix(fileName, ".json"); ok { + dates = append(dates, date) + } + } + return dates, nil +} + +func removeVersion(list attestationconfigapi.List, versionStr string) (removedVersions attestationconfigapi.List, err error) { + versions := list.List + for i, v := range versions { + if v == versionStr { + if i == len(versions)-1 { + removedVersions = attestationconfigapi.List{List: versions[:i], Variant: list.Variant} + } else { + removedVersions = attestationconfigapi.List{List: append(versions[:i], versions[i+1:]...), Variant: list.Variant} + } + return removedVersions, nil + } + } + return attestationconfigapi.List{}, fmt.Errorf("version %s not found in list %v", versionStr, versions) +} + +type crudCmd interface { + Execute(ctx context.Context, c *apiclient.Client) error +} + +type deleteCmd struct { + apiObject apiclient.APIObject +} + +func (d deleteCmd) Execute(ctx context.Context, c *apiclient.Client) error { + return apiclient.DeleteWithSignature(ctx, c, d.apiObject) +} + +type putCmd struct { + apiObject apiclient.APIObject + signer sigstore.Signer +} + +func (p putCmd) Execute(ctx context.Context, c *apiclient.Client) error { + return apiclient.SignAndUpdate(ctx, c, p.apiObject, p.signer) +} + +func executeAllCmds(ctx context.Context, client *apiclient.Client, cmds []crudCmd) error { + for _, cmd := range cmds { + if err := cmd.Execute(ctx, client); err != nil { + return fmt.Errorf("execute operation %+v: %w", cmd, err) + } + } + return nil +} diff --git a/internal/api/attestationconfigapi/cli/client/client_test.go b/internal/api/attestationconfigapi/cli/client/client_test.go new file mode 100644 index 000000000..f46c872d8 --- /dev/null +++ b/internal/api/attestationconfigapi/cli/client/client_test.go @@ -0,0 +1,34 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ +package client + +import ( + "testing" + + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/stretchr/testify/assert" +) + +func TestDeleteAzureSEVSNPVersions(t *testing.T) { + sut := Client{ + bucketID: "bucket", + } + versions := attestationconfigapi.List{List: []string{"2023-01-01.json", "2021-01-01.json", "2019-01-01.json"}} + + ops, err := sut.deleteVersion(versions, "2021-01-01") + + assert := assert.New(t) + assert.NoError(err) + assert.Contains(ops, deleteCmd{ + apiObject: attestationconfigapi.Entry{ + Version: "2021-01-01.json", + }, + }) + + assert.Contains(ops, putCmd{ + apiObject: attestationconfigapi.List{List: []string{"2023-01-01.json", "2019-01-01.json"}}, + }) +} diff --git a/internal/api/attestationconfigapi/cli/client/reporter.go b/internal/api/attestationconfigapi/cli/client/reporter.go new file mode 100644 index 000000000..ff215ca55 --- /dev/null +++ b/internal/api/attestationconfigapi/cli/client/reporter.go @@ -0,0 +1,367 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path" + "sort" + "strings" + "time" + + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/edgelesssys/constellation/v2/internal/api/client" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" +) + +// cachedVersionsSubDir is the subdirectory in the bucket where the cached versions are stored. +const cachedVersionsSubDir = "cached-versions" + +// ErrNoNewerVersion is returned if the input version is not newer than the latest API version. +var ErrNoNewerVersion = errors.New("input version is not newer than latest API version") + +func reportVersionDir(attestation variant.Variant) string { + return path.Join(attestationconfigapi.AttestationURLPath, attestation.String(), cachedVersionsSubDir) +} + +// IsInputNewerThanOtherVersion compares the input version with the other version and returns true if the input version is newer. +// This function panics if the input versions are not TDX or SEV-SNP versions. +func IsInputNewerThanOtherVersion(variant variant.Variant, inputVersion, otherVersion any) bool { + var result bool + actionForVariant(variant, + func() { + input := inputVersion.(attestationconfigapi.TDXVersion) + other := otherVersion.(attestationconfigapi.TDXVersion) + result = isInputNewerThanOtherTDXVersion(input, other) + }, + func() { + input := inputVersion.(attestationconfigapi.SEVSNPVersion) + other := otherVersion.(attestationconfigapi.SEVSNPVersion) + result = isInputNewerThanOtherSEVSNPVersion(input, other) + }, + ) + return result +} + +// UploadLatestVersion saves the given version to the cache, determines the smallest +// TCB version in the cache among the last cacheWindowSize versions and updates +// the latest version in the API if there is an update. +// force can be used to bypass the validation logic against the cached versions. +func (c Client) UploadLatestVersion( + ctx context.Context, attestationVariant variant.Variant, + inputVersion, latestVersionInAPI any, + now time.Time, force bool, +) error { + // Validate input versions against configured attestation variant + // This allows us to skip these checks in the individual variant implementations + var err error + actionForVariant(attestationVariant, + func() { + if _, ok := inputVersion.(attestationconfigapi.TDXVersion); !ok { + err = fmt.Errorf("input version %q is not a TDX version", inputVersion) + } + if _, ok := latestVersionInAPI.(attestationconfigapi.TDXVersion); !ok { + err = fmt.Errorf("latest API version %q is not a TDX version", latestVersionInAPI) + } + }, + func() { + if _, ok := inputVersion.(attestationconfigapi.SEVSNPVersion); !ok { + err = fmt.Errorf("input version %q is not a SNP version", inputVersion) + } + if _, ok := latestVersionInAPI.(attestationconfigapi.SEVSNPVersion); !ok { + err = fmt.Errorf("latest API version %q is not a SNP version", latestVersionInAPI) + } + }, + ) + if err != nil { + return err + } + + if err := c.addVersionToCache(ctx, attestationVariant, inputVersion, now); err != nil { + return fmt.Errorf("adding version to cache: %w", err) + } + + // If force is set, immediately update the latest version to the new version in the API. + if force { + return c.uploadAsLatestVersion(ctx, attestationVariant, inputVersion, now) + } + + // Otherwise, check the cached versions and update the latest version in the API if necessary. + versionDates, err := c.listCachedVersions(ctx, attestationVariant) + if err != nil { + return fmt.Errorf("listing existing cached versions: %w", err) + } + if len(versionDates) < c.cacheWindowSize { + c.log.Warn(fmt.Sprintf("Skipping version update, found %d, expected %d reported versions.", len(versionDates), c.cacheWindowSize)) + return nil + } + + minVersion, minDate, err := c.findMinVersion(ctx, attestationVariant, versionDates) + if err != nil { + return fmt.Errorf("determining minimal version in cache: %w", err) + } + c.log.Info(fmt.Sprintf("Found minimal version: %+v with date: %s", minVersion, minDate)) + + if !IsInputNewerThanOtherVersion(attestationVariant, minVersion, latestVersionInAPI) { + c.log.Info(fmt.Sprintf("Input version: %+v is not newer than latest API version: %+v. Skipping list update", minVersion, latestVersionInAPI)) + return ErrNoNewerVersion + } + + c.log.Info(fmt.Sprintf("Input version: %+v is newer than latest API version: %+v", minVersion, latestVersionInAPI)) + t, err := time.Parse(VersionFormat, minDate) + if err != nil { + return fmt.Errorf("parsing date: %w", err) + } + + if err := c.uploadAsLatestVersion(ctx, attestationVariant, minVersion, t); err != nil { + return fmt.Errorf("uploading as latest version: %w", err) + } + + c.log.Info(fmt.Sprintf("Successfully uploaded new %s version: %+v", attestationVariant, minVersion)) + return nil +} + +// uploadAsLatestVersion uploads the given version and updates the list to set it as the "latest" version. +// The version's name is the UTC timestamp of the date. +// The /list entry stores the version name + .json suffix. +func (c Client) uploadAsLatestVersion(ctx context.Context, variant variant.Variant, inputVersion any, date time.Time) error { + versions, err := c.List(ctx, variant) + if err != nil { + return fmt.Errorf("fetch version list: %w", err) + } + if !variant.Equal(versions.Variant) { + return nil + } + + dateStr := date.Format(VersionFormat) + ".json" + var ops []crudCmd + + obj := apiVersionObject{version: dateStr, variant: variant, cached: false} + obj.setVersion(inputVersion) + ops = append(ops, putCmd{ + apiObject: obj, + signer: c.signer, + }) + + versions.AddVersion(dateStr) + + ops = append(ops, putCmd{ + apiObject: versions, + signer: c.signer, + }) + + return executeAllCmds(ctx, c.s3Client, ops) +} + +// addVersionToCache adds the given version to the cache. +func (c Client) addVersionToCache(ctx context.Context, variant variant.Variant, inputVersion any, date time.Time) error { + dateStr := date.Format(VersionFormat) + ".json" + obj := apiVersionObject{version: dateStr, variant: variant, cached: true} + obj.setVersion(inputVersion) + cmd := putCmd{ + apiObject: obj, + signer: c.signer, + } + return cmd.Execute(ctx, c.s3Client) +} + +// findMinVersion returns the minimal version in the cache among the last cacheWindowSize versions. +func (c Client) findMinVersion( + ctx context.Context, attestationVariant variant.Variant, versionDates []string, +) (any, string, error) { + var getMinimalVersion func() (any, string, error) + + actionForVariant(attestationVariant, + func() { + getMinimalVersion = func() (any, string, error) { + return findMinimalVersion[attestationconfigapi.TDXVersion](ctx, attestationVariant, versionDates, c.s3Client, c.cacheWindowSize) + } + }, + func() { + getMinimalVersion = func() (any, string, error) { + return findMinimalVersion[attestationconfigapi.SEVSNPVersion](ctx, attestationVariant, versionDates, c.s3Client, c.cacheWindowSize) + } + }, + ) + return getMinimalVersion() +} + +func findMinimalVersion[T attestationconfigapi.TDXVersion | attestationconfigapi.SEVSNPVersion]( + ctx context.Context, variant variant.Variant, versionDates []string, + s3Client *client.Client, cacheWindowSize int, +) (T, string, error) { + var minimalVersion *T + var minimalDate string + sort.Sort(sort.Reverse(sort.StringSlice(versionDates))) // sort in reverse order to slice the latest versions + versionDates = versionDates[:cacheWindowSize] + sort.Strings(versionDates) // sort with oldest first to to take the minimal version with the oldest date + + for _, date := range versionDates { + obj, err := client.Fetch(ctx, s3Client, apiVersionObject{version: date + ".json", variant: variant, cached: true}) + if err != nil { + return *new(T), "", fmt.Errorf("get object: %w", err) + } + obj.variant = variant // variant is not set by Fetch, set it manually + + if minimalVersion == nil { + v := obj.getVersion().(T) + minimalVersion = &v + minimalDate = date + continue + } + + // If the current minimal version has newer versions than the one we just fetched, + // update the minimal version to the older version. + if IsInputNewerThanOtherVersion(variant, *minimalVersion, obj.getVersion()) { + v := obj.getVersion().(T) + minimalVersion = &v + minimalDate = date + } + } + + return *minimalVersion, minimalDate, nil +} + +type apiVersionObject struct { + version string `json:"-"` + variant variant.Variant `json:"-"` + cached bool `json:"-"` + snp attestationconfigapi.SEVSNPVersion + tdx attestationconfigapi.TDXVersion +} + +func (a apiVersionObject) MarshalJSON() ([]byte, error) { + var res []byte + var err error + actionForVariant(a.variant, + func() { + res, err = json.Marshal(a.tdx) + }, + func() { + res, err = json.Marshal(a.snp) + }, + ) + return res, err +} + +func (a *apiVersionObject) UnmarshalJSON(data []byte) error { + errTDX := json.Unmarshal(data, &a.tdx) + errSNP := json.Unmarshal(data, &a.snp) + if errTDX == nil || errSNP == nil { + return nil + } + return fmt.Errorf("trying to unmarshal data into both TDX and SNP versions: %w", errors.Join(errTDX, errSNP)) +} + +// JSONPath returns the path to the JSON file for the request to the config api. +// This is the path to the cached version in the S3 bucket. +func (a apiVersionObject) JSONPath() string { + if a.cached { + return path.Join(reportVersionDir(a.variant), a.version) + } + return path.Join(attestationconfigapi.AttestationURLPath, a.variant.String(), a.version) +} + +// ValidateRequest validates the request. +func (a apiVersionObject) ValidateRequest() error { + if !strings.HasSuffix(a.version, ".json") { + return fmt.Errorf("version has no .json suffix") + } + return nil +} + +// Validate is a No-Op. +func (a apiVersionObject) Validate() error { + return nil +} + +// getVersion returns the version. +func (a apiVersionObject) getVersion() any { + var res any + actionForVariant(a.variant, + func() { + res = a.tdx + }, + func() { + res = a.snp + }, + ) + return res +} + +// setVersion sets the version. +func (a *apiVersionObject) setVersion(version any) { + actionForVariant(a.variant, + func() { + a.tdx = version.(attestationconfigapi.TDXVersion) + }, + func() { + a.snp = version.(attestationconfigapi.SEVSNPVersion) + }, + ) +} + +// actionForVariant performs the given action based on the whether variant is a TDX or SEV-SNP variant. +func actionForVariant( + attestationVariant variant.Variant, + tdxAction func(), snpAction func(), +) { + switch attestationVariant { + case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}: + snpAction() + case variant.AzureTDX{}: + tdxAction() + default: + panic(fmt.Sprintf("unsupported attestation variant: %s", attestationVariant)) + } +} + +// isInputNewerThanOtherSEVSNPVersion compares all version fields and returns false if any input field is older, or the versions are equal. +func isInputNewerThanOtherSEVSNPVersion(input, other attestationconfigapi.SEVSNPVersion) bool { + if input == other { + return false + } + if input.TEE < other.TEE { + return false + } + if input.SNP < other.SNP { + return false + } + if input.Microcode < other.Microcode { + return false + } + if input.Bootloader < other.Bootloader { + return false + } + return true +} + +// isInputNewerThanOtherSEVSNPVersion compares all version fields and returns false if any input field is older, or the versions are equal. +func isInputNewerThanOtherTDXVersion(input, other attestationconfigapi.TDXVersion) bool { + if input == other { + return false + } + + if input.PCESVN < other.PCESVN { + return false + } + if input.QESVN < other.QESVN { + return false + } + + // Validate component-wise security version numbers + for idx, inputVersion := range input.TEETCBSVN { + if inputVersion < other.TEETCBSVN[idx] { + return false + } + } + + return true +} diff --git a/internal/api/attestationconfigapi/cli/client/reporter_test.go b/internal/api/attestationconfigapi/cli/client/reporter_test.go new file mode 100644 index 000000000..ded25b020 --- /dev/null +++ b/internal/api/attestationconfigapi/cli/client/reporter_test.go @@ -0,0 +1,138 @@ +/* +Copyright (c) Edgeless Systems GmbH +SPDX-License-Identifier: BUSL-1.1 +*/ +package client + +import ( + "testing" + + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/stretchr/testify/assert" +) + +func TestIsInputNewerThanOtherSEVSNPVersion(t *testing.T) { + newTestCfg := func() attestationconfigapi.SEVSNPVersion { + return attestationconfigapi.SEVSNPVersion{ + Microcode: 93, + TEE: 0, + SNP: 6, + Bootloader: 2, + } + } + + testCases := map[string]struct { + latest attestationconfigapi.SEVSNPVersion + input attestationconfigapi.SEVSNPVersion + expect bool + }{ + "input is older than latest": { + input: func(c attestationconfigapi.SEVSNPVersion) attestationconfigapi.SEVSNPVersion { + c.Microcode-- + return c + }(newTestCfg()), + latest: newTestCfg(), + expect: false, + }, + "input has greater and smaller version field than latest": { + input: func(c attestationconfigapi.SEVSNPVersion) attestationconfigapi.SEVSNPVersion { + c.Microcode++ + c.Bootloader-- + return c + }(newTestCfg()), + latest: newTestCfg(), + expect: false, + }, + "input is newer than latest": { + input: func(c attestationconfigapi.SEVSNPVersion) attestationconfigapi.SEVSNPVersion { + c.TEE++ + return c + }(newTestCfg()), + latest: newTestCfg(), + expect: true, + }, + "input is equal to latest": { + input: newTestCfg(), + latest: newTestCfg(), + expect: false, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + isNewer := isInputNewerThanOtherSEVSNPVersion(tc.input, tc.latest) + assert.Equal(t, tc.expect, isNewer) + }) + } +} + +func TestIsInputNewerThanOtherTDXVersion(t *testing.T) { + newTestVersion := func() attestationconfigapi.TDXVersion { + return attestationconfigapi.TDXVersion{ + QESVN: 1, + PCESVN: 2, + TEETCBSVN: [16]byte{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, + QEVendorID: [16]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + XFAM: [8]byte{0, 1, 2, 3, 4, 5, 6, 7}, + } + } + + testCases := map[string]struct { + latest attestationconfigapi.TDXVersion + input attestationconfigapi.TDXVersion + expect bool + }{ + "input is older than latest": { + input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion { + c.QESVN-- + return c + }(newTestVersion()), + latest: newTestVersion(), + expect: false, + }, + "input has greater and smaller version field than latest": { + input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion { + c.QESVN++ + c.PCESVN-- + return c + }(newTestVersion()), + latest: newTestVersion(), + expect: false, + }, + "input is newer than latest": { + input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion { + c.QESVN++ + return c + }(newTestVersion()), + latest: newTestVersion(), + expect: true, + }, + "input is equal to latest": { + input: newTestVersion(), + latest: newTestVersion(), + expect: false, + }, + "tee tcb svn is newer": { + input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion { + c.TEETCBSVN[4]++ + return c + }(newTestVersion()), + latest: newTestVersion(), + expect: true, + }, + "xfam is different": { + input: func(c attestationconfigapi.TDXVersion) attestationconfigapi.TDXVersion { + c.XFAM[3]++ + return c + }(newTestVersion()), + latest: newTestVersion(), + expect: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + isNewer := isInputNewerThanOtherTDXVersion(tc.input, tc.latest) + assert.Equal(t, tc.expect, isNewer) + }) + } +} diff --git a/internal/api/attestationconfigapi/cli/compare.go b/internal/api/attestationconfigapi/cli/compare.go new file mode 100644 index 000000000..36113c689 --- /dev/null +++ b/internal/api/attestationconfigapi/cli/compare.go @@ -0,0 +1,101 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ +package main + +import ( + "fmt" + "os" + "slices" + + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi/cli/client" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/verify" + "github.com/google/go-tdx-guest/proto/tdx" + "github.com/spf13/afero" + "github.com/spf13/cobra" +) + +func newCompareCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "compare VARIANT FILE [FILE...]", + Short: "Returns the minimum version of all given attestation reports.", + Long: "Compare a list of attestation reports and return the report with the minimum version.", + Example: "cli compare azure-sev-snp report1.json report2.json", + Args: cobra.MatchAll(cobra.MinimumNArgs(2), arg0isAttestationVariant()), + RunE: runCompare, + } + + return cmd +} + +func runCompare(cmd *cobra.Command, args []string) error { + cmd.SetOut(os.Stdout) + + variant, err := variant.FromString(args[0]) + if err != nil { + return fmt.Errorf("parsing variant: %w", err) + } + + return compare(cmd, variant, args[1:], file.NewHandler(afero.NewOsFs())) +} + +func compare(cmd *cobra.Command, attestationVariant variant.Variant, files []string, fs file.Handler) (retErr error) { + if !slices.Contains([]variant.Variant{variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}, variant.AzureTDX{}}, attestationVariant) { + return fmt.Errorf("variant %s not supported", attestationVariant) + } + + lowestVersion, err := compareVersions(attestationVariant, files, fs) + if err != nil { + return fmt.Errorf("comparing versions: %w", err) + } + + cmd.Println(lowestVersion) + return nil +} + +func compareVersions(attestationVariant variant.Variant, files []string, fs file.Handler) (string, error) { + readReport := readSNPReport + if attestationVariant.Equal(variant.AzureTDX{}) { + readReport = readTDXReport + } + + lowestVersion := files[0] + lowestReport, err := readReport(files[0], fs) + if err != nil { + return "", fmt.Errorf("reading report: %w", err) + } + + for _, file := range files[1:] { + report, err := readReport(file, fs) + if err != nil { + return "", fmt.Errorf("reading report: %w", err) + } + + if client.IsInputNewerThanOtherVersion(attestationVariant, lowestReport, report) { + lowestVersion = file + lowestReport = report + } + } + + return lowestVersion, nil +} + +func readSNPReport(file string, fs file.Handler) (any, error) { + var report verify.Report + if err := fs.ReadJSON(file, &report); err != nil { + return nil, fmt.Errorf("reading snp report: %w", err) + } + return convertTCBVersionToSNPVersion(report.SNPReport.LaunchTCB), nil +} + +func readTDXReport(file string, fs file.Handler) (any, error) { + var report *tdx.QuoteV4 + if err := fs.ReadJSON(file, &report); err != nil { + return nil, fmt.Errorf("reading tdx report: %w", err) + } + return convertQuoteToTDXVersion(report), nil +} diff --git a/internal/api/attestationconfigapi/cli/delete.go b/internal/api/attestationconfigapi/cli/delete.go index faedabd11..6c4b6ca81 100644 --- a/internal/api/attestationconfigapi/cli/delete.go +++ b/internal/api/attestationconfigapi/cli/delete.go @@ -1,42 +1,46 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main import ( + "context" "errors" "fmt" + "log/slog" "path" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi/cli/client" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/staticupload" "github.com/spf13/cobra" - "go.uber.org/zap" ) // newDeleteCmd creates the delete command. func newDeleteCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "delete {azure|aws} {snp-report|guest-firmware} ", + Use: "delete VARIANT KIND ", Short: "Delete an object from the attestationconfig API", Long: "Delete a specific object version from the config api. is the name of the object to delete (without .json suffix)", - Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli delete azure snp-report 1.0.0", - Args: cobra.MatchAll(cobra.ExactArgs(3), isCloudProvider(0), isValidKind(1)), + Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli delete azure-sev-snp attestation-report 1.0.0", + Args: cobra.MatchAll(cobra.ExactArgs(3), arg0isAttestationVariant(), isValidKind(1)), PreRunE: envCheck, RunE: runDelete, } recursivelyCmd := &cobra.Command{ - Use: "recursive {azure|aws}", + Use: "recursive {aws-sev-snp|azure-sev-snp|azure-tdx|gcp-sev-snp}", Short: "delete all objects from the API path constellation/v1/attestation/", Long: "Delete all objects from the API path constellation/v1/attestation/", - Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli delete recursive azure", - Args: cobra.MatchAll(cobra.ExactArgs(1), isCloudProvider(0)), + Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli delete recursive azure-sev-snp", + Args: cobra.MatchAll(cobra.ExactArgs(1), arg0isAttestationVariant()), RunE: runRecursiveDelete, } @@ -46,7 +50,7 @@ func newDeleteCmd() *cobra.Command { } func runDelete(cmd *cobra.Command, args []string) (retErr error) { - log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi") + log := logger.NewTextLogger(slog.LevelDebug).WithGroup("attestationconfigapi") deleteCfg, err := newDeleteConfig(cmd, ([3]string)(args[:3])) if err != nil { @@ -58,7 +62,7 @@ func runDelete(cmd *cobra.Command, args []string) (retErr error) { Region: deleteCfg.region, DistributionID: deleteCfg.distribution, } - client, clientClose, err := attestationconfigapi.NewClient(cmd.Context(), cfg, + client, clientClose, err := client.New(cmd.Context(), cfg, []byte(cosignPwd), []byte(privateKey), false, 1, log) if err != nil { return fmt.Errorf("create attestation client: %w", err) @@ -70,14 +74,7 @@ func runDelete(cmd *cobra.Command, args []string) (retErr error) { } }() - switch deleteCfg.provider { - case cloudprovider.AWS: - return deleteAWS(cmd.Context(), client, deleteCfg) - case cloudprovider.Azure: - return deleteAzure(cmd.Context(), client, deleteCfg) - default: - return fmt.Errorf("unsupported cloud provider: %s", deleteCfg.provider) - } + return deleteEntry(cmd.Context(), client, deleteCfg) } func runRecursiveDelete(cmd *cobra.Command, args []string) (retErr error) { @@ -89,7 +86,7 @@ func runRecursiveDelete(cmd *cobra.Command, args []string) (retErr error) { return fmt.Errorf("creating delete config: %w", err) } - log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi") + log := logger.NewTextLogger(slog.LevelDebug).WithGroup("attestationconfigapi") client, closeFn, err := staticupload.New(cmd.Context(), staticupload.Config{ Bucket: deleteCfg.bucket, Region: deleteCfg.region, @@ -105,21 +102,13 @@ func runRecursiveDelete(cmd *cobra.Command, args []string) (retErr error) { } }() - var deletePath string - switch deleteCfg.provider { - case cloudprovider.AWS: - deletePath = path.Join(attestationconfigapi.AttestationURLPath, variant.AWSSEVSNP{}.String()) - case cloudprovider.Azure: - deletePath = path.Join(attestationconfigapi.AttestationURLPath, variant.AzureSEVSNP{}.String()) - default: - return fmt.Errorf("unsupported cloud provider: %s", deleteCfg.provider) - } + deletePath := path.Join(attestationconfigapi.AttestationURLPath, deleteCfg.variant.String()) - return deleteRecursive(cmd.Context(), deletePath, client, deleteCfg) + return deleteEntryRecursive(cmd.Context(), deletePath, client, deleteCfg) } type deleteConfig struct { - provider cloudprovider.Provider + variant variant.Variant kind objectKind version string region string @@ -146,12 +135,15 @@ func newDeleteConfig(cmd *cobra.Command, args [3]string) (deleteConfig, error) { } apiCfg := getAPIEnvironment(testing) - provider := cloudprovider.FromString(args[0]) + variant, err := variant.FromString(args[0]) + if err != nil { + return deleteConfig{}, fmt.Errorf("invalid attestation variant: %q: %w", args[0], err) + } kind := kindFromString(args[1]) version := args[2] return deleteConfig{ - provider: provider, + variant: variant, kind: kind, version: version, region: region, @@ -161,3 +153,44 @@ func newDeleteConfig(cmd *cobra.Command, args [3]string) (deleteConfig, error) { cosignPublicKey: apiCfg.cosignPublicKey, }, nil } + +func deleteEntry(ctx context.Context, client *client.Client, cfg deleteConfig) error { + if cfg.kind != attestationReport { + return fmt.Errorf("kind %s not supported", cfg.kind) + } + + return client.DeleteVersion(ctx, cfg.variant, cfg.version) +} + +func deleteEntryRecursive(ctx context.Context, path string, client *staticupload.Client, cfg deleteConfig) error { + resp, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(cfg.bucket), + Prefix: aws.String(path), + }) + if err != nil { + return err + } + + // Delete all objects in the path. + objIDs := make([]s3types.ObjectIdentifier, len(resp.Contents)) + for i, obj := range resp.Contents { + objIDs[i] = s3types.ObjectIdentifier{Key: obj.Key} + } + if len(objIDs) > 0 { + _, err = client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(cfg.bucket), + Delete: &s3types.Delete{ + Objects: objIDs, + Quiet: toPtr(true), + }, + }) + if err != nil { + return err + } + } + return nil +} + +func toPtr[T any](v T) *T { + return &v +} diff --git a/internal/api/attestationconfigapi/cli/e2e/test.sh.in b/internal/api/attestationconfigapi/cli/e2e/test.sh.in index 773443df4..647fd6e08 100755 --- a/internal/api/attestationconfigapi/cli/e2e/test.sh.in +++ b/internal/api/attestationconfigapi/cli/e2e/test.sh.in @@ -19,36 +19,67 @@ configapi_cli=$(realpath @@CONFIGAPI_CLI@@) stat "${configapi_cli}" >> /dev/null configapi_cli="${configapi_cli} --testing" ###### script body ###### -function variant() { - if [[ $1 == "aws" ]]; then - echo "aws-sev-snp" - return 0 - elif [[ $1 == "azure" ]]; then - echo "azure-sev-snp" - return 0 - else - echo "Unknown CSP: $1" - exit 1 - fi -} - -csp=$1 -readonly csp -attestationType=$(variant "$csp") +attestationVariant=$1 +readonly attestationVariant readonly region="eu-west-1" readonly bucket="resource-api-testing" tmpdir=$(mktemp -d) readonly tmpdir -registerExitHandler "rm -rf $tmpdir" +registerExitHandler "rm -rf ${tmpdir}" # empty the bucket version state -${configapi_cli} delete recursive "$csp" --region "$region" --bucket "$bucket" +${configapi_cli} delete recursive "${attestationVariant}" --region "${region}" --bucket "${bucket}" -# the high version numbers ensure that it's newer than the current latest value -readonly current_report_path="$tmpdir/currentSnpReport.json" -cat << EOF > "$current_report_path" +readonly current_report_path="${tmpdir}/attestationReportCurrent.json" +readonly report_path="${tmpdir}/attestationReport.json" +readonly older_report_path="${tmpdir}/attestationReportOld.json" + +if [[ ${attestationVariant} == *-tdx ]]; then + cat << EOF > "${current_report_path}" +{ + "header": { + "qe_svn": "AAA=", + "pce_svn": "AAA=", + "qe_vendor_id": "KioqKioqKioqKioqKioqKg==" + }, + "td_quote_body": { + "tee_tcb_svn": "AAAAAAAAAAAAAAAAAAAAAA==", + "xfam": "AAAAAAAAAAA=" + } +} +EOF + # the high version numbers ensure that it's newer than the current latest value + cat << EOF > "${report_path}" +{ + "header": { + "qe_svn": "//8=", + "pce_svn": "//8=", + "qe_vendor_id": "KioqKioqKioqKioqKioqKg==" + }, + "td_quote_body": { + "tee_tcb_svn": "/////////////////////w==", + "xfam": "AQIDBAUGBwg=" + } +} +EOF + # has an older version + cat << EOF > "${older_report_path}" +{ + "header": { + "qe_svn": "//8=", + "pce_svn": "/v8=", + "qe_vendor_id": "KioqKioqKioqKioqKioqKg==" + }, + "td_quote_body": { + "tee_tcb_svn": "/////////////////////g==", + "xfam": "AQIDBAUGBwg=" + } +} +EOF +elif [[ ${attestationVariant} == *-sev-snp ]]; then + cat << EOF > "${current_report_path}" { "snp_report": { "reported_tcb": { @@ -72,90 +103,105 @@ cat << EOF > "$current_report_path" } } EOF + # the high version numbers ensure that it's newer than the current latest value + cat << EOF > "${report_path}" +{ + "snp_report": { + "reported_tcb": { + "bootloader": 255, + "tee": 255, + "snp": 255, + "microcode": 255 + }, + "committed_tcb": { + "bootloader": 255, + "tee": 255, + "snp": 255, + "microcode": 255 + }, + "launch_tcb": { + "bootloader": 255, + "tee": 255, + "snp": 255, + "microcode": 255 + } + } +} +EOF + # has an older version + cat << EOF > "${older_report_path}" +{ + "snp_report": { + "reported_tcb": { + "bootloader": 255, + "tee": 255, + "snp": 255, + "microcode": 254 + }, + "committed_tcb": { + "bootloader": 255, + "tee": 255, + "snp": 255, + "microcode": 254 + }, + "launch_tcb": { + "bootloader": 255, + "tee": 255, + "snp": 255, + "microcode": 254 + } + } +} +EOF +else + echo "Unknown attestation variant: ${attestationVariant}" + exit 1 +fi + # upload a fake latest version for the fetcher -${configapi_cli} upload "$csp" snp-report "$current_report_path" --force --upload-date "2000-01-01-01-01" --region "$region" --bucket "$bucket" - -# the high version numbers ensure that it's newer than the current latest value -readonly report_path="$tmpdir/snpReport.json" -cat << EOF > "$report_path" -{ - "snp_report": { - "reported_tcb": { - "bootloader": 255, - "tee": 255, - "snp": 255, - "microcode": 255 - }, - "committed_tcb": { - "bootloader": 255, - "tee": 255, - "snp": 255, - "microcode": 255 - }, - "launch_tcb": { - "bootloader": 255, - "tee": 255, - "snp": 255, - "microcode": 255 - } - } -} -EOF - -# has an older version -readonly older_report_path="$tmpdir/snpReportOld.json" -cat << EOF > "$older_report_path" -{ - "snp_report": { - "reported_tcb": { - "bootloader": 255, - "tee": 255, - "snp": 255, - "microcode": 254 - }, - "committed_tcb": { - "bootloader": 255, - "tee": 255, - "snp": 255, - "microcode": 254 - }, - "launch_tcb": { - "bootloader": 255, - "tee": 255, - "snp": 255, - "microcode": 254 - } - } -} -EOF +${configapi_cli} upload "${attestationVariant}" attestation-report "${current_report_path}" --force --upload-date "2000-01-01-01-01" --region "${region}" --bucket "${bucket}" # report 3 versions with different dates to fill the reporter cache readonly date_oldest="2023-02-01-03-04" -${configapi_cli} upload "$csp" snp-report "$older_report_path" --upload-date "$date_oldest" --region "$region" --bucket "$bucket" --cache-window-size 3 +${configapi_cli} upload "${attestationVariant}" attestation-report "${older_report_path}" --upload-date "${date_oldest}" --region "${region}" --bucket "${bucket}" --cache-window-size 3 readonly date_older="2023-02-02-03-04" -${configapi_cli} upload "$csp" snp-report "$older_report_path" --upload-date "$date_older" --region "$region" --bucket "$bucket" --cache-window-size 3 +${configapi_cli} upload "${attestationVariant}" attestation-report "${older_report_path}" --upload-date "${date_older}" --region "${region}" --bucket "${bucket}" --cache-window-size 3 readonly date="2023-02-03-03-04" -${configapi_cli} upload "$csp" snp-report "$report_path" --upload-date "$date" --region "$region" --bucket "$bucket" --cache-window-size 3 +${configapi_cli} upload "${attestationVariant}" attestation-report "${report_path}" --upload-date "${date}" --region "${region}" --bucket "${bucket}" --cache-window-size 3 # expect that $date_oldest is served as latest version -basepath="constellation/v1/attestation/${attestationType}" +basepath="constellation/v1/attestation/${attestationVariant}" baseurl="https://d33dzgxuwsgbpw.cloudfront.net/${basepath}" -if ! curl -fsSL "${baseurl}"/${date_oldest}.json > version.json; then +if ! curl -fsSL "${baseurl}/${date_oldest}.json" > version.json; then echo "Checking for uploaded version file ${basepath}/${date_oldest}.json: request returned ${?}" exit 1 fi -# check that version values are equal to expected -if ! cmp -s <(echo -n '{"bootloader":255,"tee":255,"snp":255,"microcode":254}') version.json; then - echo "The version content:" - cat version.json - echo " is not equal to the expected version content:" - echo '{"bootloader":255,"tee":255,"snp":255,"microcode":254}' - exit 1 + +if [[ ${attestationVariant} == *-tdx ]]; then + # check that version values are equal to expected + if ! cmp -s <(echo -n '{"qeSVN":65535,"pceSVN":65534,"teeTCBSVN":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,254],"qeVendorID":[42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42],"xfam":[1,2,3,4,5,6,7,8]}') version.json; then + echo "The version content:" + cat version.json + echo " is not equal to the expected version content:" + echo '{"qeSVN":65535,"pceSVN":65534,"teeTCBSVN":[255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,254],"qeVendorID":[42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42],"xfam":[1,2,3,4,5,6,7,8]}' + exit 1 + fi +elif [[ ${attestationVariant} == *-sev-snp ]]; then + # check that version values are equal to expected + if ! cmp -s <(echo -n '{"bootloader":255,"tee":255,"snp":255,"microcode":254}') version.json; then + echo "The version content:" + cat version.json + echo " is not equal to the expected version content:" + echo '{"bootloader":255,"tee":255,"snp":255,"microcode":254}' + exit 1 + fi fi -if ! curl -fsSL "${baseurl}"/${date_oldest}.json.sig > /dev/null; then + +if ! curl -fsSL "${baseurl}/${date_oldest}.json.sig" > /dev/null; then echo "Checking for uploaded version signature file ${basepath}/${date_oldest}.json.sig: request returned ${?}" exit 1 fi + # check list endpoint if ! curl -fsSL "${baseurl}"/list > list.json; then echo "Checking for uploaded list file ${basepath}/list: request returned ${?}" @@ -171,28 +217,28 @@ if ! cmp -s <(echo -n '["2023-02-01-03-04.json","2000-01-01-01-01.json"]') list. fi # check that the other versions are not uploaded -http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}"/${date_older}.json) -if [[ $http_code -ne 404 ]]; then +http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date_older}.json") +if [[ ${http_code} -ne 404 ]]; then echo "Expected HTTP code 404 for: ${basepath}/${date_older}.json, but got ${http_code}" exit 1 fi -http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}"/${date}.json.sig) -if [[ $http_code -ne 404 ]]; then +http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date}.json.sig") +if [[ ${http_code} -ne 404 ]]; then echo "Expected HTTP code 404 for: ${basepath}/${date}.json, but got ${http_code}" exit 1 fi -${configapi_cli} delete "$csp" snp-report "$date_oldest" --region "$region" --bucket "$bucket" +${configapi_cli} delete "${attestationVariant}" attestation-report "${date_oldest}" --region "${region}" --bucket "${bucket}" # Omit -f to check for 404. We want to check that a file was deleted, therefore we expect the query to fail. -http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}"/${date_oldest}.json) -if [[ $http_code -ne 404 ]]; then +http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date_oldest}.json") +if [[ ${http_code} -ne 404 ]]; then echo "Expected HTTP code 404 for: ${basepath}/${date_oldest}.json, but got ${http_code}" exit 1 fi # Omit -f to check for 404. We want to check that a file was deleted, therefore we expect the query to fail. -http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}"/${date_oldest}.json.sig) -if [[ $http_code -ne 404 ]]; then +http_code=$(curl -sSL -w '%{http_code}\n' -o /dev/null "${baseurl}/${date_oldest}.json.sig") +if [[ ${http_code} -ne 404 ]]; then echo "Expected HTTP code 404 for: ${basepath}/${date_oldest}.json, but got ${http_code}" exit 1 fi diff --git a/internal/api/attestationconfigapi/cli/main.go b/internal/api/attestationconfigapi/cli/main.go index e6e951f1b..76a0e6aef 100644 --- a/internal/api/attestationconfigapi/cli/main.go +++ b/internal/api/attestationconfigapi/cli/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -27,8 +27,11 @@ const ( distributionID = constants.CDNDefaultDistributionID envCosignPwd = "COSIGN_PASSWORD" envCosignPrivateKey = "COSIGN_PRIVATE_KEY" - // versionWindowSize defines the number of versions to be considered for the latest version. Each week 5 versions are uploaded for each node of the verify cluster. - versionWindowSize = 15 + // versionWindowSize defines the number of versions to be considered for the latest version. + // Through our weekly e2e tests, each week 2 versions are uploaded: + // One from a stable release, and one from a debug image. + // A window size of 6 ensures we update only after a version has been "stable" for 3 weeks. + versionWindowSize = 6 ) var ( @@ -56,6 +59,7 @@ func newRootCmd() *cobra.Command { rootCmd.AddCommand(newUploadCmd()) rootCmd.AddCommand(newDeleteCmd()) + rootCmd.AddCommand(newCompareCmd()) return rootCmd } diff --git a/internal/api/attestationconfigapi/cli/upload.go b/internal/api/attestationconfigapi/cli/upload.go index 831f99da7..dbfba690c 100644 --- a/internal/api/attestationconfigapi/cli/upload.go +++ b/internal/api/attestationconfigapi/cli/upload.go @@ -1,44 +1,47 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main import ( "context" + "encoding/binary" "errors" "fmt" + "log/slog" "os" "time" "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi/cli/client" + "github.com/edgelesssys/constellation/v2/internal/api/fetcher" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/staticupload" "github.com/edgelesssys/constellation/v2/internal/verify" + "github.com/google/go-tdx-guest/proto/tdx" "github.com/spf13/afero" "github.com/spf13/cobra" - "go.uber.org/zap" ) func newUploadCmd() *cobra.Command { uploadCmd := &cobra.Command{ - Use: "upload {azure|aws} {snp-report|guest-firmware} ", + Use: "upload VARIANT KIND FILE", Short: "Upload an object to the attestationconfig API", - Long: fmt.Sprintf("Upload a new object to the attestationconfig API. For snp-reports the new object is added to a cache folder first."+ + Long: fmt.Sprintf("Upload a new object to the attestationconfig API. For snp-reports the new object is added to a cache folder first.\n"+ "The CLI then determines the lowest version within the cache-window present in the cache and writes that value to the config api if necessary. "+ - "For guest-firmware objects the object is added to the API directly. "+ - "Please authenticate with AWS through your preferred method (e.g. environment variables, CLI)"+ + "For guest-firmware objects the object is added to the API directly.\n"+ + "Please authenticate with AWS through your preferred method (e.g. environment variables, CLI) "+ "to be able to upload to S3. Set the %s and %s environment variables to authenticate with cosign.", envCosignPrivateKey, envCosignPwd, ), - Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli upload azure snp-report /some/path/report.json", + Example: "COSIGN_PASSWORD=$CPW COSIGN_PRIVATE_KEY=$CKEY cli upload azure-sev-snp attestation-report /some/path/report.json", - Args: cobra.MatchAll(cobra.ExactArgs(3), isCloudProvider(0), isValidKind(1)), + Args: cobra.MatchAll(cobra.ExactArgs(3), arg0isAttestationVariant(), isValidKind(1)), PreRunE: envCheck, RunE: runUpload, } @@ -61,25 +64,22 @@ func envCheck(_ *cobra.Command, _ []string) error { func runUpload(cmd *cobra.Command, args []string) (retErr error) { ctx := cmd.Context() - log := logger.New(logger.PlainLog, zap.DebugLevel).Named("attestationconfigapi") + log := logger.NewTextLogger(slog.LevelDebug).WithGroup("attestationconfigapi") uploadCfg, err := newConfig(cmd, ([3]string)(args[:3])) if err != nil { return fmt.Errorf("parsing cli flags: %w", err) } - client, clientClose, err := attestationconfigapi.NewClient( - ctx, + client, clientClose, err := client.New(ctx, staticupload.Config{ Bucket: uploadCfg.bucket, Region: uploadCfg.region, DistributionID: uploadCfg.distribution, }, - []byte(cosignPwd), - []byte(privateKey), - false, - uploadCfg.cacheWindowSize, - log) + []byte(cosignPwd), []byte(privateKey), + false, uploadCfg.cacheWindowSize, log, + ) defer func() { err := clientClose(cmd.Context()) @@ -92,54 +92,57 @@ func runUpload(cmd *cobra.Command, args []string) (retErr error) { return fmt.Errorf("creating client: %w", err) } - var attesation variant.Variant - switch uploadCfg.provider { - case cloudprovider.AWS: - attesation = variant.AWSSEVSNP{} - case cloudprovider.Azure: - attesation = variant.AzureSEVSNP{} - default: - return fmt.Errorf("unsupported cloud provider: %s", uploadCfg.provider) - } - - return uploadReport(ctx, attesation, client, uploadCfg, file.NewHandler(afero.NewOsFs()), log) + return uploadReport(ctx, client, uploadCfg, file.NewHandler(afero.NewOsFs()), log) } -func uploadReport(ctx context.Context, - attestation variant.Variant, - client *attestationconfigapi.Client, - cfg uploadConfig, - fs file.Handler, - log *logger.Logger, +func uploadReport( + ctx context.Context, apiClient *client.Client, + cfg uploadConfig, fs file.Handler, log *slog.Logger, ) error { - if cfg.kind != snpReport { + if cfg.kind != attestationReport { return fmt.Errorf("kind %s not supported", cfg.kind) } - log.Infof("Reading SNP report from file: %s", cfg.path) - var report verify.Report - if err := fs.ReadJSON(cfg.path, &report); err != nil { - return fmt.Errorf("reading snp report: %w", err) - } - - inputVersion := convertTCBVersionToSNPVersion(report.SNPReport.LaunchTCB) - log.Infof("Input report: %+v", inputVersion) - - latestAPIVersionAPI, err := attestationconfigapi.NewFetcherWithCustomCDNAndCosignKey(cfg.url, cfg.cosignPublicKey).FetchSEVSNPVersionLatest(ctx, attestation) + apiFetcher := attestationconfigapi.NewFetcherWithCustomCDNAndCosignKey(cfg.url, cfg.cosignPublicKey) + latestVersionInAPI, err := apiFetcher.FetchLatestVersion(ctx, cfg.variant) if err != nil { - if errors.Is(err, attestationconfigapi.ErrNoVersionsFound) { - log.Infof("No versions found in API, but assuming that we are uploading the first version.") + var notFoundErr *fetcher.NotFoundError + if errors.As(err, ¬FoundErr) { + log.Info("No versions found in API, but assuming that we are uploading the first version.") } else { return fmt.Errorf("fetching latest version: %w", err) } } - latestAPIVersion := latestAPIVersionAPI.SEVSNPVersion - if err := client.UploadSEVSNPVersionLatest(ctx, attestation, inputVersion, latestAPIVersion, cfg.uploadDate, cfg.force); err != nil { - if errors.Is(err, attestationconfigapi.ErrNoNewerVersion) { - log.Infof("Input version: %+v is not newer than latest API version: %+v", inputVersion, latestAPIVersion) - return nil + var newVersion, latestVersion any + switch cfg.variant { + case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.GCPSEVSNP{}: + latestVersion = latestVersionInAPI.SEVSNPVersion + + log.Info(fmt.Sprintf("Reading SNP report from file: %s", cfg.path)) + newVersion, err = readSNPReport(cfg.path, fs) + if err != nil { + return err } + log.Info(fmt.Sprintf("Input SNP report: %+v", newVersion)) + + case variant.AzureTDX{}: + latestVersion = latestVersionInAPI.TDXVersion + + log.Info(fmt.Sprintf("Reading TDX report from file: %s", cfg.path)) + newVersion, err = readTDXReport(cfg.path, fs) + if err != nil { + return err + } + log.Info(fmt.Sprintf("Input TDX report: %+v", newVersion)) + + default: + return fmt.Errorf("variant %s not supported", cfg.variant) + } + + if err := apiClient.UploadLatestVersion( + ctx, cfg.variant, newVersion, latestVersion, cfg.uploadDate, cfg.force, + ); err != nil && !errors.Is(err, client.ErrNoNewerVersion) { return fmt.Errorf("updating latest version: %w", err) } @@ -155,8 +158,18 @@ func convertTCBVersionToSNPVersion(tcb verify.TCBVersion) attestationconfigapi.S } } +func convertQuoteToTDXVersion(quote *tdx.QuoteV4) attestationconfigapi.TDXVersion { + return attestationconfigapi.TDXVersion{ + QESVN: binary.LittleEndian.Uint16(quote.Header.QeSvn), + PCESVN: binary.LittleEndian.Uint16(quote.Header.PceSvn), + QEVendorID: [16]byte(quote.Header.QeVendorId), + XFAM: [8]byte(quote.TdQuoteBody.Xfam), + TEETCBSVN: [16]byte(quote.TdQuoteBody.TeeTcbSvn), + } +} + type uploadConfig struct { - provider cloudprovider.Provider + variant variant.Variant kind objectKind path string uploadDate time.Time @@ -176,7 +189,7 @@ func newConfig(cmd *cobra.Command, args [3]string) (uploadConfig, error) { } uploadDate := time.Now() if dateStr != "" { - uploadDate, err = time.Parse(attestationconfigapi.VersionFormat, dateStr) + uploadDate, err = time.Parse(client.VersionFormat, dateStr) if err != nil { return uploadConfig{}, fmt.Errorf("parsing date: %w", err) } @@ -208,12 +221,16 @@ func newConfig(cmd *cobra.Command, args [3]string) (uploadConfig, error) { return uploadConfig{}, fmt.Errorf("getting cache window size: %w", err) } - provider := cloudprovider.FromString(args[0]) + variant, err := variant.FromString(args[0]) + if err != nil { + return uploadConfig{}, fmt.Errorf("invalid attestation variant: %q: %w", args[0], err) + } + kind := kindFromString(args[1]) path := args[2] return uploadConfig{ - provider: provider, + variant: variant, kind: kind, path: path, uploadDate: uploadDate, diff --git a/internal/api/attestationconfigapi/cli/validargs.go b/internal/api/attestationconfigapi/cli/validargs.go index c077c58e0..85a7f72dd 100644 --- a/internal/api/attestationconfigapi/cli/validargs.go +++ b/internal/api/attestationconfigapi/cli/validargs.go @@ -1,32 +1,39 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main import ( + "errors" "fmt" "strings" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/spf13/cobra" ) -func isCloudProvider(arg int) cobra.PositionalArgs { - return func(cmd *cobra.Command, args []string) error { - if provider := cloudprovider.FromString(args[arg]); provider == cloudprovider.Unknown { - return fmt.Errorf("argument %s isn't a valid cloud provider", args[arg]) +func arg0isAttestationVariant() cobra.PositionalArgs { + return func(_ *cobra.Command, args []string) error { + attestationVariant, err := variant.FromString(args[0]) + if err != nil { + return errors.New("argument 0 isn't a valid attestation variant") + } + switch attestationVariant { + case variant.AWSSEVSNP{}, variant.AzureSEVSNP{}, variant.AzureTDX{}, variant.GCPSEVSNP{}: + return nil + default: + return errors.New("argument 0 isn't a supported attestation variant") } - return nil } } func isValidKind(arg int) cobra.PositionalArgs { - return func(cmd *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { if kind := kindFromString(args[arg]); kind == unknown { - return fmt.Errorf("argument %s isn't a valid kind", args[arg]) + return fmt.Errorf("argument %s isn't a valid kind: must be one of [%q, %q]", args[arg], attestationReport, guestFirmware) } return nil } @@ -37,15 +44,15 @@ type objectKind string const ( // unknown is the default objectKind and does nothing. - unknown objectKind = "unknown-kind" - snpReport objectKind = "snp-report" - guestFirmware objectKind = "guest-firmware" + unknown objectKind = "unknown-kind" + attestationReport objectKind = "attestation-report" + guestFirmware objectKind = "guest-firmware" ) func kindFromString(s string) objectKind { lower := strings.ToLower(s) switch objectKind(lower) { - case snpReport, guestFirmware: + case attestationReport, guestFirmware: return objectKind(lower) default: return unknown diff --git a/internal/api/attestationconfigapi/client.go b/internal/api/attestationconfigapi/client.go deleted file mode 100644 index 9b4575a4c..000000000 --- a/internal/api/attestationconfigapi/client.go +++ /dev/null @@ -1,182 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ -package attestationconfigapi - -import ( - "context" - "errors" - "fmt" - "time" - - apiclient "github.com/edgelesssys/constellation/v2/internal/api/client" - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/sigstore" - - "github.com/edgelesssys/constellation/v2/internal/staticupload" -) - -// VersionFormat is the format of the version name in the S3 bucket. -const VersionFormat = "2006-01-02-15-04" - -// Client manages (modifies) the version information for the attestation variants. -type Client struct { - s3Client *apiclient.Client - s3ClientClose func(ctx context.Context) error - bucketID string - signer sigstore.Signer - cacheWindowSize int -} - -// NewClient returns a new Client. -func NewClient(ctx context.Context, cfg staticupload.Config, cosignPwd, privateKey []byte, dryRun bool, versionWindowSize int, log *logger.Logger) (*Client, apiclient.CloseFunc, error) { - s3Client, clientClose, err := apiclient.NewClient(ctx, cfg.Region, cfg.Bucket, cfg.DistributionID, dryRun, log) - if err != nil { - return nil, nil, fmt.Errorf("failed to create s3 storage: %w", err) - } - - repo := &Client{ - s3Client: s3Client, - s3ClientClose: clientClose, - signer: sigstore.NewSigner(cosignPwd, privateKey), - bucketID: cfg.Bucket, - cacheWindowSize: versionWindowSize, - } - return repo, clientClose, nil -} - -// uploadSEVSNPVersion uploads the latest version numbers of the Azure SEVSNP. Then version name is the UTC timestamp of the date. The /list entry stores the version name + .json suffix. -func (a Client) uploadSEVSNPVersion(ctx context.Context, attestation variant.Variant, version SEVSNPVersion, date time.Time) error { - versions, err := a.List(ctx, attestation) - if err != nil { - return fmt.Errorf("fetch version list: %w", err) - } - ops := a.constructUploadCmd(attestation, version, versions, date) - - return executeAllCmds(ctx, a.s3Client, ops) -} - -// DeleteSEVSNPVersion deletes the given version (without .json suffix) from the API. -func (a Client) DeleteSEVSNPVersion(ctx context.Context, attestation variant.Variant, versionStr string) error { - versions, err := a.List(ctx, attestation) - if err != nil { - return fmt.Errorf("fetch version list: %w", err) - } - - ops, err := a.deleteSEVSNPVersion(versions, versionStr) - if err != nil { - return err - } - return executeAllCmds(ctx, a.s3Client, ops) -} - -// List returns the list of versions for the given attestation variant. -func (a Client) List(ctx context.Context, attestation variant.Variant) (SEVSNPVersionList, error) { - if !attestation.Equal(variant.AzureSEVSNP{}) && !attestation.Equal(variant.AWSSEVSNP{}) { - return SEVSNPVersionList{}, fmt.Errorf("unsupported attestation variant: %s", attestation) - } - - versions, err := apiclient.Fetch(ctx, a.s3Client, SEVSNPVersionList{variant: attestation}) - if err != nil { - var notFoundErr *apiclient.NotFoundError - if errors.As(err, ¬FoundErr) { - return SEVSNPVersionList{variant: attestation}, nil - } - return SEVSNPVersionList{}, err - } - - versions.variant = attestation - - return versions, nil -} - -func (a Client) deleteSEVSNPVersion(versions SEVSNPVersionList, versionStr string) (ops []crudCmd, err error) { - versionStr = versionStr + ".json" - ops = append(ops, deleteCmd{ - apiObject: SEVSNPVersionAPI{ - Variant: versions.variant, - Version: versionStr, - }, - }) - - removedVersions, err := removeVersion(versions, versionStr) - if err != nil { - return nil, err - } - ops = append(ops, putCmd{ - apiObject: removedVersions, - signer: a.signer, - }) - return ops, nil -} - -func (a Client) constructUploadCmd(attestation variant.Variant, version SEVSNPVersion, versionNames SEVSNPVersionList, date time.Time) []crudCmd { - if !attestation.Equal(versionNames.variant) { - return nil - } - - dateStr := date.Format(VersionFormat) + ".json" - var res []crudCmd - - res = append(res, putCmd{ - apiObject: SEVSNPVersionAPI{Version: dateStr, Variant: attestation, SEVSNPVersion: version}, - signer: a.signer, - }) - - versionNames.addVersion(dateStr) - - res = append(res, putCmd{ - apiObject: versionNames, - signer: a.signer, - }) - - return res -} - -func removeVersion(list SEVSNPVersionList, versionStr string) (removedVersions SEVSNPVersionList, err error) { - versions := list.List() - for i, v := range versions { - if v == versionStr { - if i == len(versions)-1 { - removedVersions = SEVSNPVersionList{list: versions[:i], variant: list.variant} - } else { - removedVersions = SEVSNPVersionList{list: append(versions[:i], versions[i+1:]...), variant: list.variant} - } - return removedVersions, nil - } - } - return SEVSNPVersionList{}, fmt.Errorf("version %s not found in list %v", versionStr, versions) -} - -type crudCmd interface { - Execute(ctx context.Context, c *apiclient.Client) error -} - -type deleteCmd struct { - apiObject apiclient.APIObject -} - -func (d deleteCmd) Execute(ctx context.Context, c *apiclient.Client) error { - return apiclient.DeleteWithSignature(ctx, c, d.apiObject) -} - -type putCmd struct { - apiObject apiclient.APIObject - signer sigstore.Signer -} - -func (p putCmd) Execute(ctx context.Context, c *apiclient.Client) error { - return apiclient.SignAndUpdate(ctx, c, p.apiObject, p.signer) -} - -func executeAllCmds(ctx context.Context, client *apiclient.Client, cmds []crudCmd) error { - for _, cmd := range cmds { - if err := cmd.Execute(ctx, client); err != nil { - return fmt.Errorf("execute operation %+v: %w", cmd, err) - } - } - return nil -} diff --git a/internal/api/attestationconfigapi/client_test.go b/internal/api/attestationconfigapi/client_test.go deleted file mode 100644 index 9cae1bc5a..000000000 --- a/internal/api/attestationconfigapi/client_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ -package attestationconfigapi - -import ( - "testing" - "time" - - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/stretchr/testify/assert" -) - -func TestUploadAzureSEVSNP(t *testing.T) { - sut := Client{ - bucketID: "bucket", - signer: fakeSigner{}, - } - version := SEVSNPVersion{} - date := time.Date(2023, 1, 1, 1, 1, 1, 1, time.UTC) - ops := sut.constructUploadCmd(variant.AzureSEVSNP{}, version, SEVSNPVersionList{list: []string{"2021-01-01-01-01.json", "2019-01-01-01-01.json"}, variant: variant.AzureSEVSNP{}}, date) - dateStr := "2023-01-01-01-01.json" - assert := assert.New(t) - assert.Contains(ops, putCmd{ - apiObject: SEVSNPVersionAPI{ - Variant: variant.AzureSEVSNP{}, - Version: dateStr, - SEVSNPVersion: version, - }, - signer: fakeSigner{}, - }) - assert.Contains(ops, putCmd{ - apiObject: SEVSNPVersionList{variant: variant.AzureSEVSNP{}, list: []string{"2023-01-01-01-01.json", "2021-01-01-01-01.json", "2019-01-01-01-01.json"}}, - signer: fakeSigner{}, - }) -} - -func TestDeleteAzureSEVSNPVersions(t *testing.T) { - sut := Client{ - bucketID: "bucket", - } - versions := SEVSNPVersionList{list: []string{"2023-01-01.json", "2021-01-01.json", "2019-01-01.json"}} - - ops, err := sut.deleteSEVSNPVersion(versions, "2021-01-01") - - assert := assert.New(t) - assert.NoError(err) - assert.Contains(ops, deleteCmd{ - apiObject: SEVSNPVersionAPI{ - Version: "2021-01-01.json", - }, - }) - - assert.Contains(ops, putCmd{ - apiObject: SEVSNPVersionList{list: []string{"2023-01-01.json", "2019-01-01.json"}}, - }) -} - -type fakeSigner struct{} - -func (fakeSigner) Sign(_ []byte) ([]byte, error) { - return []byte("signature"), nil -} diff --git a/internal/api/attestationconfigapi/fetcher.go b/internal/api/attestationconfigapi/fetcher.go index a54e3ebc7..e7476f1e5 100644 --- a/internal/api/attestationconfigapi/fetcher.go +++ b/internal/api/attestationconfigapi/fetcher.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package attestationconfigapi import ( "context" - "errors" "fmt" apifetcher "github.com/edgelesssys/constellation/v2/internal/api/fetcher" @@ -19,14 +18,9 @@ import ( const cosignPublicKey = constants.CosignPublicKeyReleases -// ErrNoVersionsFound is returned if no versions are found. -var ErrNoVersionsFound = errors.New("no versions found") - // Fetcher fetches config API resources without authentication. type Fetcher interface { - FetchSEVSNPVersion(ctx context.Context, version SEVSNPVersionAPI) (SEVSNPVersionAPI, error) - FetchSEVSNPVersionList(ctx context.Context, list SEVSNPVersionList) (SEVSNPVersionList, error) - FetchSEVSNPVersionLatest(ctx context.Context, attestation variant.Variant) (SEVSNPVersionAPI, error) + FetchLatestVersion(ctx context.Context, attestation variant.Variant) (Entry, error) } // fetcher fetches AttestationCfg API resources without authentication. @@ -65,47 +59,43 @@ func newFetcherWithClientAndVerifier(client apifetcher.HTTPClient, cosignVerifie return &fetcher{HTTPClient: client, verifier: cosignVerifier, cdnURL: url} } -// FetchSEVSNPVersionList fetches the version list information from the config API. -func (f *fetcher) FetchSEVSNPVersionList(ctx context.Context, list SEVSNPVersionList) (SEVSNPVersionList, error) { - // TODO(derpsteb): Replace with FetchAndVerify once we move to v2 of the config API. - fetchedList, err := apifetcher.Fetch(ctx, f.HTTPClient, f.cdnURL, list) +// FetchLatestVersion returns the latest versions of the given type. +func (f *fetcher) FetchLatestVersion(ctx context.Context, variant variant.Variant) (Entry, error) { + list, err := f.fetchVersionList(ctx, variant) if err != nil { - return list, fmt.Errorf("fetching version list: %w", err) + return Entry{}, err } - // Need to set this explicitly as the variant is not part of the marshalled JSON. - fetchedList.variant = list.variant + // latest version is first in list + return f.fetchVersion(ctx, list.List[0], variant) +} + +// fetchVersionList fetches the version list information from the config API. +func (f *fetcher) fetchVersionList(ctx context.Context, variant variant.Variant) (List, error) { + fetchedList, err := apifetcher.FetchAndVerify(ctx, f.HTTPClient, f.cdnURL, List{Variant: variant}, f.verifier) + if err != nil { + return List{}, fmt.Errorf("fetching version list: %w", err) + } + + // Set the attestation variant of the list as it is not part of the marshalled JSON retrieved by Fetch + fetchedList.Variant = variant return fetchedList, nil } -// FetchSEVSNPVersion fetches the version information from the config API. -func (f *fetcher) FetchSEVSNPVersion(ctx context.Context, version SEVSNPVersionAPI) (SEVSNPVersionAPI, error) { - fetchedVersion, err := apifetcher.FetchAndVerify(ctx, f.HTTPClient, f.cdnURL, version, f.verifier) +// fetchVersion fetches the version information from the config API. +func (f *fetcher) fetchVersion(ctx context.Context, version string, variant variant.Variant) (Entry, error) { + obj := Entry{ + Version: version, + Variant: variant, + } + fetchedVersion, err := apifetcher.FetchAndVerify(ctx, f.HTTPClient, f.cdnURL, obj, f.verifier) if err != nil { - return fetchedVersion, fmt.Errorf("fetching version %s: %w", version.Version, err) + return Entry{}, fmt.Errorf("fetching version %q: %w", version, err) } - // Need to set this explicitly as the variant is not part of the marshalled JSON. - fetchedVersion.Variant = version.Variant + // Set the attestation variant of the list as it is not part of the marshalled JSON retrieved by FetchAndVerify + fetchedVersion.Variant = variant return fetchedVersion, nil } - -// FetchSEVSNPVersionLatest returns the latest versions of the given type. -func (f *fetcher) FetchSEVSNPVersionLatest(ctx context.Context, attesation variant.Variant) (res SEVSNPVersionAPI, err error) { - list, err := f.FetchSEVSNPVersionList(ctx, SEVSNPVersionList{variant: attesation}) - if err != nil { - return res, ErrNoVersionsFound - } - - getVersionRequest := SEVSNPVersionAPI{ - Version: list.List()[0], // latest version is first in list - Variant: attesation, - } - res, err = f.FetchSEVSNPVersion(ctx, getVersionRequest) - if err != nil { - return res, err - } - return -} diff --git a/internal/api/attestationconfigapi/fetcher_test.go b/internal/api/attestationconfigapi/fetcher_test.go index cb9fd86eb..18c3a203e 100644 --- a/internal/api/attestationconfigapi/fetcher_test.go +++ b/internal/api/attestationconfigapi/fetcher_test.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package attestationconfigapi import ( "bytes" - "context" "encoding/json" "errors" "fmt" @@ -22,87 +21,112 @@ import ( ) func TestFetchLatestSEVSNPVersion(t *testing.T) { + latestVersionSNP := Entry{ + SEVSNPVersion: SEVSNPVersion{ + Microcode: 93, + TEE: 0, + SNP: 6, + Bootloader: 2, + }, + } + olderVersionSNP := Entry{ + SEVSNPVersion: SEVSNPVersion{ + Microcode: 1, + TEE: 0, + SNP: 1, + Bootloader: 1, + }, + } + latestVersionTDX := Entry{ + TDXVersion: TDXVersion{ + QESVN: 2, + PCESVN: 3, + TEETCBSVN: [16]byte{4}, + QEVendorID: [16]byte{5}, + XFAM: [8]byte{6}, + }, + } + olderVersionTDX := Entry{ + TDXVersion: TDXVersion{ + QESVN: 1, + PCESVN: 2, + TEETCBSVN: [16]byte{3}, + QEVendorID: [16]byte{4}, + XFAM: [8]byte{5}, + }, + } + latestStr := "2023-06-11-14-09.json" olderStr := "2019-01-01-01-01.json" - testcases := map[string]struct { + testCases := map[string]struct { fetcherVersions []string timeAtTest time.Time wantErr bool attestation variant.Variant - expectedVersion func() SEVSNPVersionAPI - olderVersion func() SEVSNPVersionAPI - latestVersion func() SEVSNPVersionAPI + expectedVersion Entry + olderVersion Entry + latestVersion Entry }{ - "get latest version azure": { + "get latest version azure-sev-snp": { fetcherVersions: []string{latestStr, olderStr}, attestation: variant.AzureSEVSNP{}, - expectedVersion: func() SEVSNPVersionAPI { tmp := latestVersion; tmp.Variant = variant.AzureSEVSNP{}; return tmp }, - olderVersion: func() SEVSNPVersionAPI { tmp := olderVersion; tmp.Variant = variant.AzureSEVSNP{}; return tmp }, - latestVersion: func() SEVSNPVersionAPI { tmp := latestVersion; tmp.Variant = variant.AzureSEVSNP{}; return tmp }, + expectedVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AzureSEVSNP{}; return tmp }(), + olderVersion: func() Entry { tmp := olderVersionSNP; tmp.Variant = variant.AzureSEVSNP{}; return tmp }(), + latestVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AzureSEVSNP{}; return tmp }(), }, - "get latest version aws": { + "get latest version aws-sev-snp": { fetcherVersions: []string{latestStr, olderStr}, attestation: variant.AWSSEVSNP{}, - expectedVersion: func() SEVSNPVersionAPI { tmp := latestVersion; tmp.Variant = variant.AWSSEVSNP{}; return tmp }, - olderVersion: func() SEVSNPVersionAPI { tmp := olderVersion; tmp.Variant = variant.AWSSEVSNP{}; return tmp }, - latestVersion: func() SEVSNPVersionAPI { tmp := latestVersion; tmp.Variant = variant.AWSSEVSNP{}; return tmp }, + expectedVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AWSSEVSNP{}; return tmp }(), + olderVersion: func() Entry { tmp := olderVersionSNP; tmp.Variant = variant.AWSSEVSNP{}; return tmp }(), + latestVersion: func() Entry { tmp := latestVersionSNP; tmp.Variant = variant.AWSSEVSNP{}; return tmp }(), + }, + "get latest version azure-tdx": { + fetcherVersions: []string{latestStr, olderStr}, + attestation: variant.AzureTDX{}, + expectedVersion: func() Entry { tmp := latestVersionTDX; tmp.Variant = variant.AzureTDX{}; return tmp }(), + olderVersion: func() Entry { tmp := olderVersionTDX; tmp.Variant = variant.AzureTDX{}; return tmp }(), + latestVersion: func() Entry { tmp := latestVersionTDX; tmp.Variant = variant.AzureTDX{}; return tmp }(), }, } - for name, tc := range testcases { + for name, tc := range testCases { t.Run(name, func(t *testing.T) { client := &http.Client{ Transport: &fakeConfigAPIHandler{ attestation: tc.attestation, versions: tc.fetcherVersions, latestDate: latestStr, - latestVersion: tc.latestVersion(), + latestVersion: tc.latestVersion, olderDate: olderStr, - olderVersion: tc.olderVersion(), + olderVersion: tc.olderVersion, }, } - fetcher := newFetcherWithClientAndVerifier(client, dummyVerifier{}, constants.CDNRepositoryURL) - res, err := fetcher.FetchSEVSNPVersionLatest(context.Background(), tc.attestation) + fetcher := newFetcherWithClientAndVerifier(client, stubVerifier{}, constants.CDNRepositoryURL) + res, err := fetcher.FetchLatestVersion(t.Context(), tc.attestation) assert := assert.New(t) if tc.wantErr { assert.Error(err) } else { assert.NoError(err) - assert.Equal(tc.expectedVersion(), res) + assert.Equal(tc.expectedVersion, res) } }) } } -var latestVersion = SEVSNPVersionAPI{ - SEVSNPVersion: SEVSNPVersion{ - Microcode: 93, - TEE: 0, - SNP: 6, - Bootloader: 2, - }, -} - -var olderVersion = SEVSNPVersionAPI{ - SEVSNPVersion: SEVSNPVersion{ - Microcode: 1, - TEE: 0, - SNP: 1, - Bootloader: 1, - }, -} - type fakeConfigAPIHandler struct { attestation variant.Variant versions []string latestDate string - latestVersion SEVSNPVersionAPI + latestVersion Entry olderDate string - olderVersion SEVSNPVersionAPI + olderVersion Entry } // RoundTrip resolves the request and returns a dummy response. func (f *fakeConfigAPIHandler) RoundTrip(req *http.Request) (*http.Response, error) { - if req.URL.Path == fmt.Sprintf("/constellation/v1/attestation/%s/list", f.attestation.String()) { + switch req.URL.Path { + case fmt.Sprintf("/constellation/v1/attestation/%s/list", f.attestation.String()): res := &http.Response{} bt, err := json.Marshal(f.versions) if err != nil { @@ -113,7 +137,14 @@ func (f *fakeConfigAPIHandler) RoundTrip(req *http.Request) (*http.Response, err res.Header.Set("Content-Type", "application/json") res.StatusCode = http.StatusOK return res, nil - } else if req.URL.Path == fmt.Sprintf("/constellation/v1/attestation/%s/%s", f.attestation.String(), f.latestDate) { + + case fmt.Sprintf("/constellation/v1/attestation/%s/list.sig", f.attestation.String()): + res := &http.Response{} + res.Body = io.NopCloser(bytes.NewReader([]byte("null"))) + res.StatusCode = http.StatusOK + return res, nil + + case fmt.Sprintf("/constellation/v1/attestation/%s/%s", f.attestation.String(), f.latestDate): res := &http.Response{} bt, err := json.Marshal(f.latestVersion) if err != nil { @@ -123,7 +154,7 @@ func (f *fakeConfigAPIHandler) RoundTrip(req *http.Request) (*http.Response, err res.StatusCode = http.StatusOK return res, nil - } else if req.URL.Path == fmt.Sprintf("/constellation/v1/attestation/%s/%s", f.attestation.String(), f.olderDate) { + case fmt.Sprintf("/constellation/v1/attestation/%s/%s", f.attestation.String(), f.olderDate): res := &http.Response{} bt, err := json.Marshal(f.olderVersion) if err != nil { @@ -132,13 +163,14 @@ func (f *fakeConfigAPIHandler) RoundTrip(req *http.Request) (*http.Response, err res.Body = io.NopCloser(bytes.NewReader(bt)) res.StatusCode = http.StatusOK return res, nil - } else if req.URL.Path == fmt.Sprintf("/constellation/v1/attestation/%s/%s.sig", f.attestation.String(), f.latestDate) { + + case fmt.Sprintf("/constellation/v1/attestation/%s/%s.sig", f.attestation.String(), f.latestDate): res := &http.Response{} res.Body = io.NopCloser(bytes.NewReader([]byte("null"))) res.StatusCode = http.StatusOK return res, nil - } else if req.URL.Path == fmt.Sprintf("/constellation/v1/attestation/%s/%s.sig", f.attestation.String(), f.olderDate) { + case fmt.Sprintf("/constellation/v1/attestation/%s/%s.sig", f.attestation.String(), f.olderDate): res := &http.Response{} res.Body = io.NopCloser(bytes.NewReader([]byte("null"))) res.StatusCode = http.StatusOK @@ -148,8 +180,8 @@ func (f *fakeConfigAPIHandler) RoundTrip(req *http.Request) (*http.Response, err return nil, errors.New("no endpoint found") } -type dummyVerifier struct{} +type stubVerifier struct{} -func (s dummyVerifier) VerifySignature(_, _ []byte) error { +func (s stubVerifier) VerifySignature(_, _ []byte) error { return nil } diff --git a/internal/api/attestationconfigapi/reporter.go b/internal/api/attestationconfigapi/reporter.go deleted file mode 100644 index 4cc4bcad6..000000000 --- a/internal/api/attestationconfigapi/reporter.go +++ /dev/null @@ -1,189 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -/* -The reporter contains the logic to determine a latest version for Azure SEVSNP based on cached version values observed on CVM instances. -Some code in this file (e.g. listing cached files) does not rely on dedicated API objects and instead uses the AWS SDK directly, -for no other reason than original development speed. -*/ -package attestationconfigapi - -import ( - "context" - "errors" - "fmt" - "path" - "sort" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go/aws" - - "github.com/edgelesssys/constellation/v2/internal/api/client" - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" -) - -// cachedVersionsSubDir is the subdirectory in the bucket where the cached versions are stored. -const cachedVersionsSubDir = "cached-versions" - -// ErrNoNewerVersion is returned if the input version is not newer than the latest API version. -var ErrNoNewerVersion = errors.New("input version is not newer than latest API version") - -func reportVersionDir(attestation variant.Variant) string { - return path.Join(AttestationURLPath, attestation.String(), cachedVersionsSubDir) -} - -// UploadSEVSNPVersionLatest saves the given version to the cache, determines the smallest -// TCB version in the cache among the last cacheWindowSize versions and updates -// the latest version in the API if there is an update. -// force can be used to bypass the validation logic against the cached versions. -func (c Client) UploadSEVSNPVersionLatest(ctx context.Context, attestation variant.Variant, inputVersion, - latestAPIVersion SEVSNPVersion, now time.Time, force bool, -) error { - if err := c.cacheSEVSNPVersion(ctx, attestation, inputVersion, now); err != nil { - return fmt.Errorf("reporting version: %w", err) - } - if force { - return c.uploadSEVSNPVersion(ctx, attestation, inputVersion, now) - } - versionDates, err := c.listCachedVersions(ctx, attestation) - if err != nil { - return fmt.Errorf("list reported versions: %w", err) - } - if len(versionDates) < c.cacheWindowSize { - c.s3Client.Logger.Warnf("Skipping version update, found %d, expected %d reported versions.", len(versionDates), c.cacheWindowSize) - return nil - } - minVersion, minDate, err := c.findMinVersion(ctx, attestation, versionDates) - if err != nil { - return fmt.Errorf("get minimal version: %w", err) - } - c.s3Client.Logger.Infof("Found minimal version: %+v with date: %s", minVersion, minDate) - shouldUpdateAPI, err := isInputNewerThanOtherVersion(minVersion, latestAPIVersion) - if err != nil { - return ErrNoNewerVersion - } - if !shouldUpdateAPI { - c.s3Client.Logger.Infof("Input version: %+v is not newer than latest API version: %+v", minVersion, latestAPIVersion) - return nil - } - c.s3Client.Logger.Infof("Input version: %+v is newer than latest API version: %+v", minVersion, latestAPIVersion) - t, err := time.Parse(VersionFormat, minDate) - if err != nil { - return fmt.Errorf("parsing date: %w", err) - } - if err := c.uploadSEVSNPVersion(ctx, attestation, minVersion, t); err != nil { - return fmt.Errorf("uploading version: %w", err) - } - c.s3Client.Logger.Infof("Successfully uploaded new Azure SEV-SNP version: %+v", minVersion) - return nil -} - -// cacheSEVSNPVersion uploads the latest observed version numbers of the Azure SEVSNP. This version is used to later report the latest version numbers to the API. -func (c Client) cacheSEVSNPVersion(ctx context.Context, attestation variant.Variant, version SEVSNPVersion, date time.Time) error { - dateStr := date.Format(VersionFormat) + ".json" - res := putCmd{ - apiObject: reportedSEVSNPVersionAPI{Version: dateStr, variant: attestation, SEVSNPVersion: version}, - signer: c.signer, - } - return res.Execute(ctx, c.s3Client) -} - -func (c Client) listCachedVersions(ctx context.Context, attestation variant.Variant) ([]string, error) { - list, err := c.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(c.bucketID), - Prefix: aws.String(reportVersionDir(attestation)), - }) - if err != nil { - return nil, fmt.Errorf("list objects: %w", err) - } - var dates []string - for _, obj := range list.Contents { - fileName := path.Base(*obj.Key) - if strings.HasSuffix(fileName, ".json") { - dates = append(dates, fileName[:len(fileName)-5]) - } - } - return dates, nil -} - -// findMinVersion finds the minimal version of the given version dates among the latest values in the version window size. -func (c Client) findMinVersion(ctx context.Context, attesation variant.Variant, versionDates []string) (SEVSNPVersion, string, error) { - var minimalVersion *SEVSNPVersion - var minimalDate string - sort.Sort(sort.Reverse(sort.StringSlice(versionDates))) // sort in reverse order to slice the latest versions - versionDates = versionDates[:c.cacheWindowSize] - sort.Strings(versionDates) // sort with oldest first to to take the minimal version with the oldest date - for _, date := range versionDates { - obj, err := client.Fetch(ctx, c.s3Client, reportedSEVSNPVersionAPI{Version: date + ".json", variant: attesation}) - if err != nil { - return SEVSNPVersion{}, "", fmt.Errorf("get object: %w", err) - } - // Need to set this explicitly as the variant is not part of the marshalled JSON. - obj.variant = attesation - - if minimalVersion == nil { - minimalVersion = &obj.SEVSNPVersion - minimalDate = date - } else { - shouldUpdateMinimal, err := isInputNewerThanOtherVersion(*minimalVersion, obj.SEVSNPVersion) - if err != nil { - continue - } - if shouldUpdateMinimal { - minimalVersion = &obj.SEVSNPVersion - minimalDate = date - } - } - } - return *minimalVersion, minimalDate, nil -} - -// isInputNewerThanOtherVersion compares all version fields and returns true if any input field is newer. -func isInputNewerThanOtherVersion(input, other SEVSNPVersion) (bool, error) { - if input == other { - return false, nil - } - if input.TEE < other.TEE { - return false, fmt.Errorf("input TEE version: %d is older than latest API version: %d", input.TEE, other.TEE) - } - if input.SNP < other.SNP { - return false, fmt.Errorf("input SNP version: %d is older than latest API version: %d", input.SNP, other.SNP) - } - if input.Microcode < other.Microcode { - return false, fmt.Errorf("input Microcode version: %d is older than latest API version: %d", input.Microcode, other.Microcode) - } - if input.Bootloader < other.Bootloader { - return false, fmt.Errorf("input Bootloader version: %d is older than latest API version: %d", input.Bootloader, other.Bootloader) - } - return true, nil -} - -// reportedSEVSNPVersionAPI is the request to get the version information of the specific version in the config api. -type reportedSEVSNPVersionAPI struct { - Version string `json:"-"` - variant variant.Variant `json:"-"` - SEVSNPVersion -} - -// JSONPath returns the path to the JSON file for the request to the config api. -func (i reportedSEVSNPVersionAPI) JSONPath() string { - return path.Join(reportVersionDir(i.variant), i.Version) -} - -// ValidateRequest validates the request. -func (i reportedSEVSNPVersionAPI) ValidateRequest() error { - if !strings.HasSuffix(i.Version, ".json") { - return fmt.Errorf("version has no .json suffix") - } - return nil -} - -// Validate is a No-Op at the moment. -func (i reportedSEVSNPVersionAPI) Validate() error { - return nil -} diff --git a/internal/api/attestationconfigapi/reporter_test.go b/internal/api/attestationconfigapi/reporter_test.go deleted file mode 100644 index ea37d2d2f..000000000 --- a/internal/api/attestationconfigapi/reporter_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only -*/ -package attestationconfigapi - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestIsInputNewerThanLatestAPI(t *testing.T) { - newTestCfg := func() SEVSNPVersion { - return SEVSNPVersion{ - Microcode: 93, - TEE: 0, - SNP: 6, - Bootloader: 2, - } - } - - testCases := map[string]struct { - latest SEVSNPVersion - input SEVSNPVersion - expect bool - errMsg string - }{ - "input is older than latest": { - input: func(c SEVSNPVersion) SEVSNPVersion { - c.Microcode-- - return c - }(newTestCfg()), - latest: newTestCfg(), - expect: false, - errMsg: "input Microcode version: 92 is older than latest API version: 93", - }, - "input has greater and smaller version field than latest": { - input: func(c SEVSNPVersion) SEVSNPVersion { - c.Microcode++ - c.Bootloader-- - return c - }(newTestCfg()), - latest: newTestCfg(), - expect: false, - errMsg: "input Bootloader version: 1 is older than latest API version: 2", - }, - "input is newer than latest": { - input: func(c SEVSNPVersion) SEVSNPVersion { - c.TEE++ - return c - }(newTestCfg()), - latest: newTestCfg(), - expect: true, - }, - "input is equal to latest": { - input: newTestCfg(), - latest: newTestCfg(), - expect: false, - }, - } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - isNewer, err := isInputNewerThanOtherVersion(tc.input, tc.latest) - assert := assert.New(t) - if tc.errMsg != "" { - assert.EqualError(err, tc.errMsg) - } else { - assert.NoError(err) - assert.Equal(tc.expect, isNewer) - } - }) - } -} diff --git a/internal/api/attestationconfigapi/snp.go b/internal/api/attestationconfigapi/snp.go deleted file mode 100644 index 68098a3ad..000000000 --- a/internal/api/attestationconfigapi/snp.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package attestationconfigapi - -import ( - "encoding/json" - "fmt" - "path" - "sort" - "strings" - - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" -) - -// AttestationURLPath is the URL path to the attestation versions. -const AttestationURLPath = "constellation/v1/attestation" - -// SEVSNPVersion tracks the latest version of each component of the Azure SEVSNP. -type SEVSNPVersion struct { - // Bootloader is the latest version of the Azure SEVSNP bootloader. - Bootloader uint8 `json:"bootloader"` - // TEE is the latest version of the Azure SEVSNP TEE. - TEE uint8 `json:"tee"` - // SNP is the latest version of the Azure SEVSNP SNP. - SNP uint8 `json:"snp"` - // Microcode is the latest version of the Azure SEVSNP microcode. - Microcode uint8 `json:"microcode"` -} - -// SEVSNPVersionAPI is the request to get the version information of the specific version in the config api. -// Because variant is not part of the marshalled JSON, fetcher and client methods need to fill the variant property. -// Once we switch to v2 of the API we should embed the variant in the object. -// That would remove the possibility of some fetcher/client code forgetting to set the variant. -type SEVSNPVersionAPI struct { - Version string `json:"-"` - Variant variant.Variant `json:"-"` - SEVSNPVersion -} - -// JSONPath returns the path to the JSON file for the request to the config api. -func (i SEVSNPVersionAPI) JSONPath() string { - return path.Join(AttestationURLPath, i.Variant.String(), i.Version) -} - -// ValidateRequest validates the request. -func (i SEVSNPVersionAPI) ValidateRequest() error { - if !strings.HasSuffix(i.Version, ".json") { - return fmt.Errorf("version has no .json suffix") - } - return nil -} - -// Validate is a No-Op at the moment. -func (i SEVSNPVersionAPI) Validate() error { - return nil -} - -// SEVSNPVersionList is the request to list all versions in the config api. -// Because variant is not part of the marshalled JSON, fetcher and client methods need to fill the variant property. -// Once we switch to v2 of the API we could embed the variant in the object and remove some code from fetcher & client. -// That would remove the possibility of some fetcher/client code forgetting to set the variant. -type SEVSNPVersionList struct { - variant variant.Variant - list []string -} - -// MarshalJSON marshals the i's list property to JSON. -func (i SEVSNPVersionList) MarshalJSON() ([]byte, error) { - return json.Marshal(i.list) -} - -// UnmarshalJSON unmarshals a list of strings into i's list property. -func (i *SEVSNPVersionList) UnmarshalJSON(data []byte) error { - return json.Unmarshal(data, &i.list) -} - -// List returns i's list property. -func (i SEVSNPVersionList) List() []string { return i.list } - -// JSONPath returns the path to the JSON file for the request to the config api. -func (i SEVSNPVersionList) JSONPath() string { - return path.Join(AttestationURLPath, i.variant.String(), "list") -} - -// ValidateRequest is a NoOp as there is no input. -func (i SEVSNPVersionList) ValidateRequest() error { - return nil -} - -// SortReverse sorts the list of versions in reverse order. -func (i *SEVSNPVersionList) SortReverse() { - sort.Sort(sort.Reverse(sort.StringSlice(i.list))) -} - -// addVersion adds new to i's list and sorts the element in descending order. -func (i *SEVSNPVersionList) addVersion(new string) { - i.list = append(i.list, new) - i.list = variant.RemoveDuplicate(i.list) - - i.SortReverse() -} - -// Validate validates the response. -func (i SEVSNPVersionList) Validate() error { - if len(i.list) < 1 { - return fmt.Errorf("no versions found in /list") - } - return nil -} diff --git a/internal/api/attestationconfigapi/version.go b/internal/api/attestationconfigapi/version.go new file mode 100644 index 000000000..82d218872 --- /dev/null +++ b/internal/api/attestationconfigapi/version.go @@ -0,0 +1,127 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package attestationconfigapi + +import ( + "encoding/json" + "fmt" + "path" + "sort" + "strings" + + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" +) + +// AttestationURLPath is the URL path to the attestation versions. +const AttestationURLPath = "constellation/v1/attestation" + +// SEVSNPVersion tracks the latest version of each component for SEV-SNP. +type SEVSNPVersion struct { + // Bootloader is the latest version of the SEV-SNP bootloader. + Bootloader uint8 `json:"bootloader"` + // TEE is the latest version of the SEV-SNP TEE. + TEE uint8 `json:"tee"` + // SNP is the latest version of the SEV-SNP SNP. + SNP uint8 `json:"snp"` + // Microcode is the latest version of the SEV-SNP microcode. + Microcode uint8 `json:"microcode"` +} + +// TDXVersion tracks the latest version of each component for TDX. +type TDXVersion struct { + // QESVN is the latest QE security version number. + QESVN uint16 `json:"qeSVN"` + // PCESVN is the latest PCE security version number. + PCESVN uint16 `json:"pceSVN"` + // TEETCBSVN are the latest component-wise security version numbers for the TEE. + TEETCBSVN [16]byte `json:"teeTCBSVN"` + // QEVendorID is the latest QE vendor ID. + QEVendorID [16]byte `json:"qeVendorID"` + // XFAM is the latest XFAM field. + XFAM [8]byte `json:"xfam"` +} + +// Entry is the request to get the version information of the specific version in the config api. +// +// TODO: Because variant is not part of the marshalled JSON, fetcher and client methods need to fill the variant property. +// In API v2 we should embed the variant in the object and remove some code from fetcher & client. +// That would remove the possibility of some fetcher/client code forgetting to set the variant. +type Entry struct { + Version string `json:"-"` + Variant variant.Variant `json:"-"` + SEVSNPVersion + TDXVersion +} + +// JSONPath returns the path to the JSON file for the request to the config api. +func (i Entry) JSONPath() string { + return path.Join(AttestationURLPath, i.Variant.String(), i.Version) +} + +// ValidateRequest validates the request. +func (i Entry) ValidateRequest() error { + if !strings.HasSuffix(i.Version, ".json") { + return fmt.Errorf("version has no .json suffix") + } + return nil +} + +// Validate is a No-Op at the moment. +func (i Entry) Validate() error { + return nil +} + +// List is the request to retrieve of all versions in the API for one attestation variant. +// +// TODO: Because variant is not part of the marshalled JSON, fetcher and client methods need to fill the variant property. +// In API v2 we should embed the variant in the object and remove some code from fetcher & client. +// That would remove the possibility of some fetcher/client code forgetting to set the variant. +type List struct { + Variant variant.Variant + List []string +} + +// MarshalJSON marshals the i's list property to JSON. +func (i List) MarshalJSON() ([]byte, error) { + return json.Marshal(i.List) +} + +// UnmarshalJSON unmarshals a list of strings into i's list property. +func (i *List) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &i.List) +} + +// JSONPath returns the path to the JSON file for the request to the config api. +func (i List) JSONPath() string { + return path.Join(AttestationURLPath, i.Variant.String(), "list") +} + +// ValidateRequest is a NoOp as there is no input. +func (i List) ValidateRequest() error { + return nil +} + +// SortReverse sorts the list of versions in reverse order. +func (i *List) SortReverse() { + sort.Sort(sort.Reverse(sort.StringSlice(i.List))) +} + +// AddVersion adds new to i's list and sorts the element in descending order. +func (i *List) AddVersion(ver string) { + i.List = append(i.List, ver) + i.List = variant.RemoveDuplicate(i.List) + + i.SortReverse() +} + +// Validate validates the response. +func (i List) Validate() error { + if len(i.List) < 1 { + return fmt.Errorf("no versions found in /list") + } + return nil +} diff --git a/internal/api/attestationconfigapi/snp_test.go b/internal/api/attestationconfigapi/version_test.go similarity index 59% rename from internal/api/attestationconfigapi/snp_test.go rename to internal/api/attestationconfigapi/version_test.go index 2fe3ea8c9..57d80b5fa 100644 --- a/internal/api/attestationconfigapi/snp_test.go +++ b/internal/api/attestationconfigapi/version_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package attestationconfigapi @@ -14,23 +14,23 @@ import ( "github.com/stretchr/testify/require" ) -func TestSEVSNPVersionListMarshalUnmarshalJSON(t *testing.T) { +func TestVersionListMarshalUnmarshalJSON(t *testing.T) { tests := map[string]struct { - input SEVSNPVersionList - output SEVSNPVersionList + input List + output List wantDiff bool }{ "success": { - input: SEVSNPVersionList{list: []string{"v1", "v2"}}, - output: SEVSNPVersionList{list: []string{"v1", "v2"}}, + input: List{List: []string{"v1", "v2"}}, + output: List{List: []string{"v1", "v2"}}, }, "variant is lost": { - input: SEVSNPVersionList{list: []string{"v1", "v2"}, variant: variant.AzureSEVSNP{}}, - output: SEVSNPVersionList{list: []string{"v1", "v2"}}, + input: List{List: []string{"v1", "v2"}, Variant: variant.AzureSEVSNP{}}, + output: List{List: []string{"v1", "v2"}}, }, "wrong order": { - input: SEVSNPVersionList{list: []string{"v1", "v2"}}, - output: SEVSNPVersionList{list: []string{"v2", "v1"}}, + input: List{List: []string{"v1", "v2"}}, + output: List{List: []string{"v2", "v1"}}, wantDiff: true, }, } @@ -40,7 +40,7 @@ func TestSEVSNPVersionListMarshalUnmarshalJSON(t *testing.T) { inputRaw, err := tc.input.MarshalJSON() require.NoError(t, err) - var actual SEVSNPVersionList + var actual List err = actual.UnmarshalJSON(inputRaw) require.NoError(t, err) @@ -53,7 +53,7 @@ func TestSEVSNPVersionListMarshalUnmarshalJSON(t *testing.T) { } } -func TestSEVSNPVersionListAddVersion(t *testing.T) { +func TestVersionListAddVersion(t *testing.T) { tests := map[string]struct { versions []string new string @@ -68,10 +68,10 @@ func TestSEVSNPVersionListAddVersion(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { - v := SEVSNPVersionList{list: tc.versions} - v.addVersion(tc.new) + v := List{List: tc.versions} + v.AddVersion(tc.new) - assert.Equal(t, tc.expected, v.list) + assert.Equal(t, tc.expected, v.List) }) } } diff --git a/internal/api/client/BUILD.bazel b/internal/api/client/BUILD.bazel index a77457c5a..a8fef5dd2 100644 --- a/internal/api/client/BUILD.bazel +++ b/internal/api/client/BUILD.bazel @@ -6,12 +6,10 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/internal/api/client", visibility = ["//:__subpackages__"], deps = [ - "//internal/logger", "//internal/sigstore", "//internal/staticupload", "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager", "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", "@com_github_aws_aws_sdk_go_v2_service_s3//types", - "@org_uber_go_zap//:zap", ], ) diff --git a/internal/api/client/client.go b/internal/api/client/client.go index eae52df31..71a05794c 100644 --- a/internal/api/client/client.go +++ b/internal/api/client/client.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -33,16 +33,15 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "strings" "time" s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/sigstore" "github.com/edgelesssys/constellation/v2/internal/staticupload" - "go.uber.org/zap" ) // Client is the a general client for all APIs. @@ -54,13 +53,13 @@ type Client struct { dirtyPaths []string // written paths to be invalidated DryRun bool // no write operations are performed - Logger *logger.Logger + log *slog.Logger } // NewReadOnlyClient creates a new read-only client. // This client can be used to fetch objects but cannot write updates. func NewReadOnlyClient(ctx context.Context, region, bucket, distributionID string, - log *logger.Logger, + log *slog.Logger, ) (*Client, CloseFunc, error) { staticUploadClient, staticUploadClientClose, err := staticupload.New(ctx, staticupload.Config{ Region: region, @@ -78,7 +77,7 @@ func NewReadOnlyClient(ctx context.Context, region, bucket, distributionID strin s3ClientClose: staticUploadClientClose, bucket: bucket, DryRun: true, - Logger: log, + log: log, } clientClose := func(ctx context.Context) error { return client.Close(ctx) @@ -89,7 +88,7 @@ func NewReadOnlyClient(ctx context.Context, region, bucket, distributionID strin // NewClient creates a new client for the versions API. func NewClient(ctx context.Context, region, bucket, distributionID string, dryRun bool, - log *logger.Logger, + log *slog.Logger, ) (*Client, CloseFunc, error) { staticUploadClient, staticUploadClientClose, err := staticupload.New(ctx, staticupload.Config{ Region: region, @@ -107,7 +106,7 @@ func NewClient(ctx context.Context, region, bucket, distributionID string, dryRu s3ClientClose: staticUploadClientClose, bucket: bucket, DryRun: dryRun, - Logger: log, + log: log, } clientClose := func(ctx context.Context) error { return client.Close(ctx) @@ -120,7 +119,7 @@ func NewClient(ctx context.Context, region, bucket, distributionID string, dryRu // It invalidates the CDN cache for all uploaded files. func (c *Client) Close(ctx context.Context) error { if c.s3ClientClose == nil { - c.Logger.Debugf("Client has no s3ClientClose") + c.log.Debug("Client has no s3ClientClose") return nil } return c.s3ClientClose(ctx) @@ -132,10 +131,10 @@ func (c *Client) DeletePath(ctx context.Context, path string) error { Bucket: &c.bucket, Prefix: &path, } - c.Logger.Debugf("Listing objects in %s", path) + c.log.Debug(fmt.Sprintf("Listing objects in %q", path)) objs := []s3types.Object{} - out := &s3.ListObjectsV2Output{IsTruncated: true} - for out.IsTruncated { + out := &s3.ListObjectsV2Output{IsTruncated: ptr(true)} + for out.IsTruncated != nil && *out.IsTruncated { var err error out, err = c.s3Client.ListObjectsV2(ctx, listIn) if err != nil { @@ -143,10 +142,10 @@ func (c *Client) DeletePath(ctx context.Context, path string) error { } objs = append(objs, out.Contents...) } - c.Logger.Debugf("Found %d objects in %s", len(objs), path) + c.log.Debug(fmt.Sprintf("Found %d objects in %q", len(objs), path)) if len(objs) == 0 { - c.Logger.Warnf("Path %s is already empty", path) + c.log.Warn(fmt.Sprintf("Path %s is already empty", path)) return nil } @@ -156,7 +155,7 @@ func (c *Client) DeletePath(ctx context.Context, path string) error { } if c.DryRun { - c.Logger.Debugf("DryRun: Deleting %d objects with IDs %v", len(objs), objIDs) + c.log.Debug(fmt.Sprintf("DryRun: Deleting %d objects with IDs %v", len(objs), objIDs)) return nil } @@ -168,7 +167,7 @@ func (c *Client) DeletePath(ctx context.Context, path string) error { Objects: objIDs, }, } - c.Logger.Debugf("Deleting %d objects in %s", len(objs), path) + c.log.Debug(fmt.Sprintf("Deleting %d objects in %q", len(objs), path)) if _, err := c.s3Client.DeleteObjects(ctx, deleteIn); err != nil { return fmt.Errorf("deleting objects in %s: %w", path, err) } @@ -198,7 +197,7 @@ func Fetch[T APIObject](ctx context.Context, c *Client, obj T) (T, error) { Key: ptr(obj.JSONPath()), } - c.Logger.Debugf("Fetching %T from s3: %s", obj, obj.JSONPath()) + c.log.Debug(fmt.Sprintf("Fetching %T from s3: %q", obj, obj.JSONPath())) out, err := c.s3Client.GetObject(ctx, in) var noSuchkey *s3types.NoSuchKey if errors.As(err, &noSuchkey) { @@ -232,7 +231,7 @@ func Update(ctx context.Context, c *Client, obj APIObject) error { } if c.DryRun { - c.Logger.With(zap.String("bucket", c.bucket), zap.String("key", obj.JSONPath()), zap.String("body", string(rawJSON))).Debugf("DryRun: s3 put object") + c.log.With(slog.String("bucket", c.bucket), slog.String("key", obj.JSONPath()), slog.String("body", string(rawJSON))).Debug("DryRun: s3 put object") return nil } @@ -244,7 +243,7 @@ func Update(ctx context.Context, c *Client, obj APIObject) error { c.dirtyPaths = append(c.dirtyPaths, "/"+obj.JSONPath()) - c.Logger.Debugf("Uploading %T to s3: %v", obj, obj.JSONPath()) + c.log.Debug(fmt.Sprintf("Uploading %T to s3: %q", obj, obj.JSONPath())) if _, err := c.Upload(ctx, in); err != nil { return fmt.Errorf("uploading %T: %w", obj, err) } @@ -307,7 +306,7 @@ func Delete(ctx context.Context, c *Client, obj APIObject) error { Key: ptr(obj.JSONPath()), } - c.Logger.Debugf("Deleting %T from s3: %s", obj, obj.JSONPath()) + c.log.Debug(fmt.Sprintf("Deleting %T from s3: %q", obj, obj.JSONPath())) if _, err := c.DeleteObject(ctx, in); err != nil { return fmt.Errorf("deleting s3 object at %s: %w", obj.JSONPath(), err) } diff --git a/internal/api/fetcher/fetcher.go b/internal/api/fetcher/fetcher.go index 2b18f9d15..d5b407467 100644 --- a/internal/api/fetcher/fetcher.go +++ b/internal/api/fetcher/fetcher.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -20,11 +20,10 @@ package fetcher import ( "context" "encoding/json" - "errors" "fmt" + "io" "net/http" "net/url" - "strings" "github.com/edgelesssys/constellation/v2/internal/sigstore" ) @@ -40,45 +39,12 @@ func NewHTTPClient() HTTPClient { // Fetch fetches the given apiObject from the public Constellation CDN. // Fetch does not require authentication. func Fetch[T apiObject](ctx context.Context, c HTTPClient, cdnURL string, obj T) (T, error) { - if err := obj.ValidateRequest(); err != nil { - return *new(T), fmt.Errorf("validating request for %T: %w", obj, err) - } - - urlObj, err := url.Parse(cdnURL) + rawObj, err := fetch(ctx, c, cdnURL, obj) if err != nil { - return *new(T), fmt.Errorf("parsing CDN root URL: %w", err) - } - urlObj.Path = obj.JSONPath() - url := urlObj.String() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) - if err != nil { - return *new(T), fmt.Errorf("creating request for %T: %w", obj, err) + return *new(T), fmt.Errorf("fetching %T: %w", obj, err) } - resp, err := c.Do(req) - if err != nil { - return *new(T), fmt.Errorf("sending request for %T: %w", obj, err) - } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - case http.StatusNotFound: - return *new(T), &NotFoundError{fmt.Errorf("requesting resource at %s returned status code 404", url)} - default: - return *new(T), fmt.Errorf("unexpected status code %d while requesting resource", resp.StatusCode) - } - - var newObj T - if err := json.NewDecoder(resp.Body).Decode(&newObj); err != nil { - return *new(T), fmt.Errorf("decoding %T: %w", obj, err) - } - - if newObj.Validate() != nil { - return *new(T), fmt.Errorf("received invalid %T: %w", newObj, newObj.Validate()) - } - - return newObj, nil + return parseObject(rawObj, obj) } // FetchAndVerify fetches the given apiObject, checks if it can fetch an accompanying signature and verifies if the signature matches the found object. @@ -86,25 +52,70 @@ func Fetch[T apiObject](ctx context.Context, c HTTPClient, cdnURL string, obj T) // FetchAndVerify uses a generic to return a new object of type T. // Otherwise the caller would have to cast the interface type to a concrete object, which could fail. func FetchAndVerify[T apiObject](ctx context.Context, c HTTPClient, cdnURL string, obj T, cosignVerifier sigstore.Verifier) (T, error) { - fetchedObj, err := Fetch(ctx, c, cdnURL, obj) + rawObj, err := fetch(ctx, c, cdnURL, obj) if err != nil { - return fetchedObj, fmt.Errorf("fetching object: %w", err) + return *new(T), fmt.Errorf("fetching %T: %w", obj, err) } - marshalledObj, err := json.Marshal(fetchedObj) + fetchedObj, err := parseObject(rawObj, obj) if err != nil { - return fetchedObj, fmt.Errorf("marshalling object: %w", err) + return fetchedObj, fmt.Errorf("parsing %T: %w", obj, err) } + signature, err := Fetch(ctx, c, cdnURL, signature{Signed: obj.JSONPath()}) if err != nil { return fetchedObj, fmt.Errorf("fetching signature: %w", err) } - err = cosignVerifier.VerifySignature(marshalledObj, signature.Signature) + err = cosignVerifier.VerifySignature(rawObj, signature.Signature) if err != nil { return fetchedObj, fmt.Errorf("verifying signature: %w", err) } return fetchedObj, nil } +func fetch[T apiObject](ctx context.Context, c HTTPClient, cdnURL string, obj T) ([]byte, error) { + if err := obj.ValidateRequest(); err != nil { + return nil, fmt.Errorf("validating request for %T: %w", obj, err) + } + + urlObj, err := url.Parse(cdnURL) + if err != nil { + return nil, fmt.Errorf("parsing CDN root URL: %w", err) + } + urlObj.Path = obj.JSONPath() + url := urlObj.String() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if err != nil { + return nil, fmt.Errorf("creating request for %T: %w", obj, err) + } + + resp, err := c.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request for %T: %w", obj, err) + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, &NotFoundError{fmt.Errorf("requesting resource at %s returned status code 404", url)} + default: + return nil, fmt.Errorf("unexpected status code %d while requesting resource", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +func parseObject[T apiObject](rawObj []byte, obj T) (T, error) { + var newObj T + if err := json.Unmarshal(rawObj, &newObj); err != nil { + return *new(T), fmt.Errorf("decoding %T: %w", obj, err) + } + if newObj.Validate() != nil { + return *new(T), fmt.Errorf("received invalid %T: %w", newObj, newObj.Validate()) + } + return newObj, nil +} + // NotFoundError is an error that is returned when a resource is not found. type NotFoundError struct { err error @@ -132,7 +143,7 @@ type apiObject interface { // signature manages the signature of a object saved at location 'Signed'. type signature struct { // Signed is the object that is signed. - Signed string `json:"-"` + Signed string `json:"signed"` // Signature is the signature of `Signed`. Signature []byte `json:"signature"` } @@ -142,12 +153,8 @@ func (s signature) JSONPath() string { return s.Signed + ".sig" } -// ValidateRequest validates the request. +// ValidateRequest is a no-op. func (s signature) ValidateRequest() error { - if !strings.HasSuffix(s.Signed, ".json") { - return errors.New("signed object missing .json suffix") - } - return nil } diff --git a/internal/api/versionsapi/BUILD.bazel b/internal/api/versionsapi/BUILD.bazel index 2f792d6c1..30fb04c21 100644 --- a/internal/api/versionsapi/BUILD.bazel +++ b/internal/api/versionsapi/BUILD.bazel @@ -20,7 +20,6 @@ go_library( "//internal/api/client", "//internal/api/fetcher", "//internal/constants", - "//internal/logger", "@org_golang_x_mod//semver", ], ) diff --git a/internal/api/versionsapi/apiconstants.go b/internal/api/versionsapi/apiconstants.go index bca2b2b4c..832d48ec4 100644 --- a/internal/api/versionsapi/apiconstants.go +++ b/internal/api/versionsapi/apiconstants.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/cli/BUILD.bazel b/internal/api/versionsapi/cli/BUILD.bazel index 76882ae22..cc720acb4 100644 --- a/internal/api/versionsapi/cli/BUILD.bazel +++ b/internal/api/versionsapi/cli/BUILD.bazel @@ -21,13 +21,12 @@ go_library( "@com_github_aws_smithy_go//:smithy-go", "@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5//:armcompute", + "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v6//:armcompute", "@com_github_googleapis_gax_go_v2//:gax-go", "@com_github_spf13_cobra//:cobra", "@com_google_cloud_go_compute//apiv1", "@com_google_cloud_go_compute//apiv1/computepb", "@org_golang_x_mod//semver", - "@org_uber_go_zap//zapcore", ], ) diff --git a/internal/api/versionsapi/cli/add.go b/internal/api/versionsapi/cli/add.go index 4efd94612..8d7782e3d 100644 --- a/internal/api/versionsapi/cli/add.go +++ b/internal/api/versionsapi/cli/add.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -10,13 +10,12 @@ import ( "context" "errors" "fmt" + "log/slog" apiclient "github.com/edgelesssys/constellation/v2/internal/api/client" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" - "golang.org/x/mod/semver" ) func newAddCmd() *cobra.Command { @@ -52,21 +51,11 @@ func runAdd(cmd *cobra.Command, _ []string) (retErr error) { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "dryRun", flags.dryRun, "kind", flags.version.Kind(), "latest", flags.latest, "ref", flags.version.Ref(), + "stream", flags.version.Stream(), "version", flags.version.Version()) - log.Debugf("Validating flags") - if err := flags.validate(log); err != nil { - return err - } - - log.Debugf("Creating version struct") - ver, err := versionsapi.NewVersion(flags.ref, flags.stream, flags.version, flags.kind) - if err != nil { - return fmt.Errorf("creating version: %w", err) - } - - log.Debugf("Creating versions API client") + log.Debug("Creating versions API client") client, clientClose, err := versionsapi.NewClient(cmd.Context(), flags.region, flags.bucket, flags.distributionID, flags.dryRun, log) if err != nil { return fmt.Errorf("creating client: %w", err) @@ -78,92 +67,92 @@ func runAdd(cmd *cobra.Command, _ []string) (retErr error) { } }() - log.Infof("Adding version") - if err := ensureVersion(cmd.Context(), client, flags.kind, ver, versionsapi.GranularityMajor, log); err != nil { + log.Info("Adding version") + if err := ensureVersion(cmd.Context(), client, flags.version, versionsapi.GranularityMajor, log); err != nil { return err } - if err := ensureVersion(cmd.Context(), client, flags.kind, ver, versionsapi.GranularityMinor, log); err != nil { + if err := ensureVersion(cmd.Context(), client, flags.version, versionsapi.GranularityMinor, log); err != nil { return err } if flags.latest { - if err := updateLatest(cmd.Context(), client, flags.kind, ver, log); err != nil { + if err := updateLatest(cmd.Context(), client, flags.version, log); err != nil { return fmt.Errorf("setting latest version: %w", err) } } - log.Infof("List major->minor URL: %s", ver.ListURL(versionsapi.GranularityMajor)) - log.Infof("List minor->patch URL: %s", ver.ListURL(versionsapi.GranularityMinor)) + log.Info(fmt.Sprintf("List major->minor URL: %s", flags.version.ListURL(versionsapi.GranularityMajor))) + log.Info(fmt.Sprintf("List minor->patch URL: %s", flags.version.ListURL(versionsapi.GranularityMinor))) return nil } -func ensureVersion(ctx context.Context, client *versionsapi.Client, kind versionsapi.VersionKind, ver versionsapi.Version, gran versionsapi.Granularity, - log *logger.Logger, +func ensureVersion(ctx context.Context, client *versionsapi.Client, ver versionsapi.Version, gran versionsapi.Granularity, + log *slog.Logger, ) error { verListReq := versionsapi.List{ Ref: ver.Ref(), Stream: ver.Stream(), Granularity: gran, Base: ver.WithGranularity(gran), - Kind: kind, + Kind: ver.Kind(), } verList, err := client.FetchVersionList(ctx, verListReq) var notFoundErr *apiclient.NotFoundError if errors.As(err, ¬FoundErr) { - log.Infof("Version list for %s versions under %q does not exist. Creating new list", gran.String(), ver.Major()) + log.Info(fmt.Sprintf("Version list for %s versions under %q does not exist. Creating new list", gran.String(), ver.Major())) verList = verListReq } else if err != nil { return fmt.Errorf("failed to list minor versions: %w", err) } - log.Debugf("%s version list: %v", gran.String(), verList) + log.Debug(fmt.Sprintf("%q version list: %v", gran.String(), verList.Versions)) insertGran := gran + 1 insertVersion := ver.WithGranularity(insertGran) if verList.Contains(insertVersion) { - log.Infof("Version %q already exists in list %v", insertVersion, verList.Versions) + log.Info(fmt.Sprintf("Version %q already exists in list %v", insertVersion, verList.Versions)) return nil } - log.Infof("Inserting %s version %q into list", insertGran.String(), insertVersion) + log.Info(fmt.Sprintf("Inserting %s version %q into list", insertGran.String(), insertVersion)) verList.Versions = append(verList.Versions, insertVersion) - log.Debugf("New %s version list: %v", gran.String(), verList) + log.Debug(fmt.Sprintf("New %q version list: %v", gran.String(), verList.Versions)) if err := client.UpdateVersionList(ctx, verList); err != nil { return fmt.Errorf("failed to add %s version: %w", gran.String(), err) } - log.Infof("Added %q to list", insertVersion) + log.Info(fmt.Sprintf("Added %q to list", insertVersion)) return nil } -func updateLatest(ctx context.Context, client *versionsapi.Client, kind versionsapi.VersionKind, ver versionsapi.Version, log *logger.Logger) error { +func updateLatest(ctx context.Context, client *versionsapi.Client, ver versionsapi.Version, log *slog.Logger) error { latest := versionsapi.Latest{ Ref: ver.Ref(), Stream: ver.Stream(), - Kind: kind, + Kind: ver.Kind(), } latest, err := client.FetchVersionLatest(ctx, latest) var notFoundErr *apiclient.NotFoundError if errors.As(err, ¬FoundErr) { - log.Debugf("Latest version for ref %q and stream %q not found", ver.Ref(), ver.Stream()) + log.Debug(fmt.Sprintf("Latest version for ref %q and stream %q not found", ver.Ref(), ver.Stream())) } else if err != nil { return fmt.Errorf("fetching latest version: %w", err) } if latest.Version == ver.Version() { - log.Infof("Version %q is already latest version", ver) + log.Info(fmt.Sprintf("Version %q is already latest version", ver.Version())) return nil } - log.Infof("Setting %q as latest version", ver) + log.Info(fmt.Sprintf("Setting %q as latest version", ver.Version())) latest = versionsapi.Latest{ Ref: ver.Ref(), Stream: ver.Stream(), Version: ver.Version(), - Kind: kind, + Kind: ver.Kind(), } if err := client.UpdateVersionLatest(ctx, latest); err != nil { return fmt.Errorf("updating latest version: %w", err) @@ -173,52 +162,13 @@ func updateLatest(ctx context.Context, client *versionsapi.Client, kind versions } type addFlags struct { - version string - stream string - ref string - release bool + version versionsapi.Version latest bool dryRun bool region string bucket string distributionID string - kind versionsapi.VersionKind - logLevel zapcore.Level -} - -func (f *addFlags) validate(log *logger.Logger) error { - if !semver.IsValid(f.version) { - return fmt.Errorf("version %q is not a valid semantic version", f.version) - } - if semver.Canonical(f.version) != f.version { - return fmt.Errorf("version %q is not a canonical semantic version", f.version) - } - - if f.ref == "" && !f.release { - return fmt.Errorf("either --ref or --release must be set") - } - - if f.kind == versionsapi.VersionKindUnknown { - return fmt.Errorf("unknown version kind %q", f.kind) - } - - if f.release { - log.Debugf("Setting ref to %q, as release flag is set", versionsapi.ReleaseRef) - f.ref = versionsapi.ReleaseRef - } else { - log.Debugf("Setting latest to true, as release flag is not set") - f.latest = true // always set latest for non-release versions - } - - if err := versionsapi.ValidateRef(f.ref); err != nil { - return fmt.Errorf("invalid ref %w", err) - } - - if err := versionsapi.ValidateStream(f.ref, f.stream); err != nil { - return fmt.Errorf("invalid stream %w", err) - } - - return nil + logLevel slog.Level } func parseAddFlags(cmd *cobra.Command) (addFlags, error) { @@ -226,7 +176,6 @@ func parseAddFlags(cmd *cobra.Command) (addFlags, error) { if err != nil { return addFlags{}, err } - ref = versionsapi.CanonicalizeRef(ref) stream, err := cmd.Flags().GetString("stream") if err != nil { return addFlags{}, err @@ -256,9 +205,9 @@ func parseAddFlags(cmd *cobra.Command) (addFlags, error) { if err != nil { return addFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } region, err := cmd.Flags().GetString("region") if err != nil { @@ -273,17 +222,24 @@ func parseAddFlags(cmd *cobra.Command) (addFlags, error) { return addFlags{}, err } + if release { + ref = versionsapi.ReleaseRef + } else { + latest = true // always set latest for non-release versions + } + + ver, err := versionsapi.NewVersion(ref, stream, version, kind) + if err != nil { + return addFlags{}, fmt.Errorf("creating version: %w", err) + } + return addFlags{ - version: version, - stream: stream, - ref: versionsapi.CanonicalizeRef(ref), - release: release, + version: ver, latest: latest, dryRun: dryRun, region: region, bucket: bucket, distributionID: distributionID, logLevel: logLevel, - kind: kind, }, nil } diff --git a/internal/api/versionsapi/cli/latest.go b/internal/api/versionsapi/cli/latest.go index 0a7740697..ca379c43f 100644 --- a/internal/api/versionsapi/cli/latest.go +++ b/internal/api/versionsapi/cli/latest.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -10,11 +10,11 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newLatestCmd() *cobra.Command { @@ -38,15 +38,15 @@ func runLatest(cmd *cobra.Command, _ []string) (retErr error) { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "ref", flags.ref, "stream", flags.stream, "json", flags.json) - log.Debugf("Validating flags") + log.Debug("Validating flags") if err := flags.validate(); err != nil { return err } - log.Debugf("Creating versions API client") + log.Debug("Creating versions API client") client, clientClose, err := versionsapi.NewReadOnlyClient(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) if err != nil { return fmt.Errorf("creating client: %w", err) @@ -58,7 +58,7 @@ func runLatest(cmd *cobra.Command, _ []string) (retErr error) { } }() - log.Debugf("Requesting latest version") + log.Debug("Requesting latest version") latest := versionsapi.Latest{ Ref: flags.ref, Stream: flags.stream, @@ -89,7 +89,7 @@ type latestFlags struct { region string bucket string distributionID string - logLevel zapcore.Level + logLevel slog.Level } func (l *latestFlags) validate() error { @@ -133,9 +133,9 @@ func parseLatestFlags(cmd *cobra.Command) (latestFlags, error) { if err != nil { return latestFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return latestFlags{ diff --git a/internal/api/versionsapi/cli/list.go b/internal/api/versionsapi/cli/list.go index 152d0c5f7..52cfc873b 100644 --- a/internal/api/versionsapi/cli/list.go +++ b/internal/api/versionsapi/cli/list.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -11,9 +11,9 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" "golang.org/x/mod/semver" apiclient "github.com/edgelesssys/constellation/v2/internal/api/client" @@ -43,15 +43,16 @@ func runList(cmd *cobra.Command, _ []string) (retErr error) { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "bucket", flags.bucket, "distributionID", flags.distributionID, "json", flags.json, "minorVersion", flags.minorVersion, + "ref", flags.ref, "region", flags.region, "stream", flags.stream) - log.Debugf("Validating flags") + log.Debug("Validating flags") if err := flags.validate(); err != nil { return err } - log.Debugf("Creating versions API client") + log.Debug("Creating versions API client") client, clientClose, err := versionsapi.NewReadOnlyClient(cmd.Context(), flags.region, flags.bucket, flags.distributionID, log) if err != nil { return fmt.Errorf("creating client: %w", err) @@ -67,34 +68,34 @@ func runList(cmd *cobra.Command, _ []string) (retErr error) { if flags.minorVersion != "" { minorVersions = []string{flags.minorVersion} } else { - log.Debugf("Getting minor versions") + log.Debug("Getting minor versions") minorVersions, err = listMinorVersions(cmd.Context(), client, flags.ref, flags.stream) var errNotFound *apiclient.NotFoundError if err != nil && errors.As(err, &errNotFound) { - log.Infof("No minor versions found for ref %q and stream %q.", flags.ref, flags.stream) + log.Info(fmt.Sprintf("No minor versions found for ref %q and stream %q.", flags.ref, flags.stream)) return nil } else if err != nil { return err } } - log.Debugf("Getting patch versions") + log.Debug("Getting patch versions") patchVersions, err := listPatchVersions(cmd.Context(), client, flags.ref, flags.stream, minorVersions) var errNotFound *apiclient.NotFoundError if err != nil && errors.As(err, &errNotFound) { - log.Infof("No patch versions found for ref %q, stream %q and minor versions %v.", flags.ref, flags.stream, minorVersions) + log.Info(fmt.Sprintf("No patch versions found for ref %q, stream %q and minor versions %v.", flags.ref, flags.stream, minorVersions)) return nil } else if err != nil { return err } if flags.json { - log.Debugf("Printing versions as JSON") + log.Debug("Printing versions as JSON") var vers []string for _, v := range patchVersions { vers = append(vers, v.Version()) } - raw, err := json.Marshal(vers) + raw, err := json.MarshalIndent(vers, "", " ") if err != nil { return fmt.Errorf("marshaling versions: %w", err) } @@ -102,7 +103,7 @@ func runList(cmd *cobra.Command, _ []string) (retErr error) { return nil } - log.Debugf("Printing versions") + log.Debug("Printing versions") for _, v := range patchVersions { fmt.Println(v.ShortPath()) } @@ -158,7 +159,7 @@ type listFlags struct { bucket string distributionID string json bool - logLevel zapcore.Level + logLevel slog.Level } func (l *listFlags) validate() error { @@ -211,9 +212,9 @@ func parseListFlags(cmd *cobra.Command) (listFlags, error) { if err != nil { return listFlags{}, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return listFlags{ diff --git a/internal/api/versionsapi/cli/main.go b/internal/api/versionsapi/cli/main.go index 0410f8064..2c173da1b 100644 --- a/internal/api/versionsapi/cli/main.go +++ b/internal/api/versionsapi/cli/main.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/api/versionsapi/cli/rm.go b/internal/api/versionsapi/cli/rm.go index f41d7510c..06757f441 100644 --- a/internal/api/versionsapi/cli/rm.go +++ b/internal/api/versionsapi/cli/rm.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -12,6 +12,7 @@ import ( "fmt" "io" "log" + "log/slog" "regexp" "strings" "time" @@ -20,7 +21,7 @@ import ( "cloud.google.com/go/compute/apiv1/computepb" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/smithy-go" @@ -29,7 +30,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/logger" gaxv2 "github.com/googleapis/gax-go/v2" "github.com/spf13/cobra" - "go.uber.org/zap/zapcore" ) func newRemoveCmd() *cobra.Command { @@ -74,33 +74,35 @@ func runRemove(cmd *cobra.Command, _ []string) (retErr error) { if err != nil { return err } - log := logger.New(logger.PlainLog, flags.logLevel) - log.Debugf("Parsed flags: %+v", flags) + log := logger.NewTextLogger(flags.logLevel) + log.Debug("Using flags", "all", flags.all, "azLocation", flags.azLocation, "azResourceGroup", flags.azResourceGroup, "azSubscription", flags.azSubscription, + "bucket", flags.bucket, "distributionID", flags.distributionID, "dryrun", flags.dryrun, "gcpProject", flags.gcpProject, "ref", flags.ref, + "region", flags.region, "stream", flags.stream, "version", flags.version, "versionPath", flags.versionPath) - log.Debugf("Validating flags") + log.Debug("Validating flags") if err := flags.validate(); err != nil { return err } - log.Debugf("Creating GCP client") + log.Debug("Creating GCP client") gcpClient, err := newGCPClient(cmd.Context(), flags.gcpProject) if err != nil { return fmt.Errorf("creating GCP client: %w", err) } - log.Debugf("Creating AWS client") + log.Debug("Creating AWS client") awsClient, err := newAWSClient() if err != nil { return fmt.Errorf("creating AWS client: %w", err) } - log.Debugf("Creating Azure client") + log.Debug("Creating Azure client") azClient, err := newAzureClient(flags.azSubscription, flags.azLocation, flags.azResourceGroup) if err != nil { return fmt.Errorf("creating Azure client: %w", err) } - log.Debugf("Creating versions API client") + log.Debug("Creating versions API client") verclient, verclientClose, err := versionsapi.NewClient(cmd.Context(), flags.region, flags.bucket, flags.distributionID, flags.dryrun, log) if err != nil { return fmt.Errorf("creating client: %w", err) @@ -120,14 +122,14 @@ func runRemove(cmd *cobra.Command, _ []string) (retErr error) { } if flags.all { - log.Infof("Deleting ref %s", flags.ref) + log.Info(fmt.Sprintf("Deleting ref %s", flags.ref)) if err := deleteRef(cmd.Context(), imageClients, flags.ref, flags.dryrun, log); err != nil { return fmt.Errorf("deleting ref: %w", err) } return nil } - log.Infof("Deleting single version %s", flags.ver.ShortPath()) + log.Info(fmt.Sprintf("Deleting single version %s", flags.ver.ShortPath())) if err := deleteSingleVersion(cmd.Context(), imageClients, flags.ver, flags.dryrun, log); err != nil { return fmt.Errorf("deleting single version: %w", err) } @@ -135,15 +137,15 @@ func runRemove(cmd *cobra.Command, _ []string) (retErr error) { return nil } -func deleteSingleVersion(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *logger.Logger) error { +func deleteSingleVersion(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *slog.Logger) error { var retErr error - log.Debugf("Deleting images for %s", ver.Version) + log.Debug(fmt.Sprintf("Deleting images for %q", ver.Version())) if err := deleteImage(ctx, clients, ver, dryrun, log); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting images: %w", err)) } - log.Debugf("Deleting version %s from versions API", ver.Version) + log.Debug(fmt.Sprintf("Deleting version %q from versions API", ver.Version())) if err := clients.version.DeleteVersion(ctx, ver); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting version from versions API: %w", err)) } @@ -151,15 +153,15 @@ func deleteSingleVersion(ctx context.Context, clients rmImageClients, ver versio return retErr } -func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun bool, log *logger.Logger) error { +func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun bool, log *slog.Logger) error { var vers []versionsapi.Version for _, stream := range []string{"nightly", "console", "debug"} { - log.Infof("Listing versions of stream %s", stream) + log.Info(fmt.Sprintf("Listing versions of stream %s", stream)) minorVersions, err := listMinorVersions(ctx, clients.version, ref, stream) var notFoundErr *apiclient.NotFoundError if errors.As(err, ¬FoundErr) { - log.Debugf("No minor versions found for stream %s", stream) + log.Debug(fmt.Sprintf("No minor versions found for stream %q", stream)) continue } else if err != nil { return fmt.Errorf("listing minor versions for stream %s: %w", stream, err) @@ -167,7 +169,7 @@ func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun b patchVersions, err := listPatchVersions(ctx, clients.version, ref, stream, minorVersions) if errors.As(err, ¬FoundErr) { - log.Debugf("No patch versions found for stream %s", stream) + log.Debug(fmt.Sprintf("No patch versions found for stream %q", stream)) continue } else if err != nil { return fmt.Errorf("listing patch versions for stream %s: %w", stream, err) @@ -175,7 +177,7 @@ func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun b vers = append(vers, patchVersions...) } - log.Infof("Found %d versions to delete", len(vers)) + log.Info(fmt.Sprintf("Found %d versions to delete", len(vers))) var retErr error @@ -185,7 +187,7 @@ func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun b } } - log.Infof("Deleting ref %s from versions API", ref) + log.Info(fmt.Sprintf("Deleting ref %s from versions API", ref)) if err := clients.version.DeleteRef(ctx, ref); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting ref from versions API: %w", err)) } @@ -193,7 +195,7 @@ func deleteRef(ctx context.Context, clients rmImageClients, ref string, dryrun b return retErr } -func deleteImage(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *logger.Logger) error { +func deleteImage(ctx context.Context, clients rmImageClients, ver versionsapi.Version, dryrun bool, log *slog.Logger) error { var retErr error imageInfo := versionsapi.ImageInfo{ @@ -204,8 +206,8 @@ func deleteImage(ctx context.Context, clients rmImageClients, ver versionsapi.Ve imageInfo, err := clients.version.FetchImageInfo(ctx, imageInfo) var notFound *apiclient.NotFoundError if errors.As(err, ¬Found) { - log.Warnf("Image info for %s not found", ver.Version) - log.Warnf("Skipping image deletion") + log.Warn(fmt.Sprintf("Image info for %s not found", ver.Version())) + log.Warn("Skipping image deletion") return nil } else if err != nil { return fmt.Errorf("fetching image info: %w", err) @@ -214,17 +216,17 @@ func deleteImage(ctx context.Context, clients rmImageClients, ver versionsapi.Ve for _, entry := range imageInfo.List { switch entry.CSP { case "aws": - log.Infof("Deleting AWS images from %s", imageInfo.JSONPath()) + log.Info(fmt.Sprintf("Deleting AWS images from %s", imageInfo.JSONPath())) if err := clients.aws.deleteImage(ctx, entry.Reference, entry.Region, dryrun, log); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting AWS image %s: %w", entry.Reference, err)) } case "gcp": - log.Infof("Deleting GCP images from %s", imageInfo.JSONPath()) + log.Info(fmt.Sprintf("Deleting GCP images from %s", imageInfo.JSONPath())) if err := clients.gcp.deleteImage(ctx, entry.Reference, dryrun, log); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting GCP image %s: %w", entry.Reference, err)) } case "azure": - log.Infof("Deleting Azure images from %s", imageInfo.JSONPath()) + log.Info(fmt.Sprintf("Deleting Azure images from %s", imageInfo.JSONPath())) if err := clients.az.deleteImage(ctx, entry.Reference, dryrun, log); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting Azure image %s: %w", entry.Reference, err)) } @@ -259,7 +261,7 @@ type rmFlags struct { azSubscription string azLocation string azResourceGroup string - logLevel zapcore.Level + logLevel slog.Level ver versionsapi.Version } @@ -358,9 +360,9 @@ func parseRmFlags(cmd *cobra.Command) (*rmFlags, error) { if err != nil { return nil, err } - logLevel := zapcore.InfoLevel + logLevel := slog.LevelInfo if verbose { - logLevel = zapcore.DebugLevel + logLevel = slog.LevelDebug } return &rmFlags{ @@ -400,17 +402,17 @@ type ec2API interface { ) (*ec2.DeleteSnapshotOutput, error) } -func (a *awsClient) deleteImage(ctx context.Context, ami string, region string, dryrun bool, log *logger.Logger) error { +func (a *awsClient) deleteImage(ctx context.Context, ami string, region string, dryrun bool, log *slog.Logger) error { cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) if err != nil { return err } a.ec2 = ec2.NewFromConfig(cfg) - log.Debugf("Deleting resources in AWS region %s", region) + log.Debug(fmt.Sprintf("Deleting resources in AWS region %q", region)) snapshotID, err := a.getSnapshotID(ctx, ami, log) if err != nil { - log.Warnf("Failed to get AWS snapshot ID for image %s: %v", ami, err) + log.Warn(fmt.Sprintf("Failed to get AWS snapshot ID for image %s: %v", ami, err)) } if err := a.deregisterImage(ctx, ami, dryrun, log); err != nil { @@ -426,8 +428,8 @@ func (a *awsClient) deleteImage(ctx context.Context, ami string, region string, return nil } -func (a *awsClient) deregisterImage(ctx context.Context, ami string, dryrun bool, log *logger.Logger) error { - log.Debugf("Deregistering image %s", ami) +func (a *awsClient) deregisterImage(ctx context.Context, ami string, dryrun bool, log *slog.Logger) error { + log.Debug(fmt.Sprintf("Deregistering image %q", ami)) deregisterReq := ec2.DeregisterImageInput{ ImageId: &ami, @@ -438,15 +440,15 @@ func (a *awsClient) deregisterImage(ctx context.Context, ami string, dryrun bool if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "InvalidAMIID.NotFound" || apiErr.ErrorCode() == "InvalidAMIID.Unavailable") { - log.Warnf("AWS image %s not found", ami) + log.Warn(fmt.Sprintf("AWS image %s not found", ami)) return nil } return err } -func (a *awsClient) getSnapshotID(ctx context.Context, ami string, log *logger.Logger) (string, error) { - log.Debugf("Describing image %s", ami) +func (a *awsClient) getSnapshotID(ctx context.Context, ami string, log *slog.Logger) (string, error) { + log.Debug(fmt.Sprintf("Describing image %q", ami)) req := ec2.DescribeImagesInput{ ImageIds: []string{ami}, @@ -481,8 +483,8 @@ func (a *awsClient) getSnapshotID(ctx context.Context, ami string, log *logger.L return snapshotID, nil } -func (a *awsClient) deleteSnapshot(ctx context.Context, snapshotID string, dryrun bool, log *logger.Logger) error { - log.Debugf("Deleting AWS snapshot %s", snapshotID) +func (a *awsClient) deleteSnapshot(ctx context.Context, snapshotID string, dryrun bool, log *slog.Logger) error { + log.Debug(fmt.Sprintf("Deleting AWS snapshot %q", snapshotID)) req := ec2.DeleteSnapshotInput{ SnapshotId: &snapshotID, @@ -493,7 +495,7 @@ func (a *awsClient) deleteSnapshot(ctx context.Context, snapshotID string, dryru if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "InvalidSnapshot.NotFound" || apiErr.ErrorCode() == "InvalidSnapshot.Unavailable") { - log.Warnf("AWS snapshot %s not found", snapshotID) + log.Warn(fmt.Sprintf("AWS snapshot %s not found", snapshotID)) return nil } @@ -523,7 +525,7 @@ type gcpComputeAPI interface { io.Closer } -func (g *gcpClient) deleteImage(ctx context.Context, imageURI string, dryrun bool, log *logger.Logger) error { +func (g *gcpClient) deleteImage(ctx context.Context, imageURI string, dryrun bool, log *slog.Logger) error { // Extract image name from image URI // Expected input into function: "projects/constellation-images/global/images/v2-6-0-stable" // Required for computepb.DeleteImageRequest: "v2-6-0-stable" @@ -536,20 +538,20 @@ func (g *gcpClient) deleteImage(ctx context.Context, imageURI string, dryrun boo } if dryrun { - log.Debugf("DryRun: delete image request: %v", req) + log.Debug(fmt.Sprintf("DryRun: delete image request: %q", req.String())) return nil } - log.Debugf("Deleting image %s", image) + log.Debug(fmt.Sprintf("Deleting image %q", image)) op, err := g.compute.Delete(ctx, req) if err != nil && strings.Contains(err.Error(), "404") { - log.Warnf("GCP image %s not found", image) + log.Warn(fmt.Sprintf("GCP image %s not found", image)) return nil } else if err != nil { return fmt.Errorf("deleting image %s: %w", image, err) } - log.Debugf("Waiting for operation to finish") + log.Debug("Waiting for operation to finish") if err := op.Wait(ctx); err != nil { return fmt.Errorf("waiting for operation: %w", err) } @@ -624,30 +626,30 @@ var ( azCommunityImageRegex = regexp.MustCompile("^/CommunityGalleries/([[:alnum:]-]+)/Images/([[:alnum:]._-]+)/Versions/([[:alnum:]._-]+)$") ) -func (a *azureClient) deleteImage(ctx context.Context, image string, dryrun bool, log *logger.Logger) error { +func (a *azureClient) deleteImage(ctx context.Context, image string, dryrun bool, log *slog.Logger) error { azImage, err := a.parseImage(ctx, image, log) if err != nil { return err } if dryrun { - log.Debugf("DryRun: delete image %v", azImage) + log.Debug(fmt.Sprintf("DryRun: delete image: gallery: %q, image definition: %q, resource group: %q, version: %q", azImage.gallery, azImage.imageDefinition, azImage.resourceGroup, azImage.version)) return nil } - log.Debugf("Deleting image %q, version %q", azImage.imageDefinition, azImage.version) + log.Debug(fmt.Sprintf("Deleting image %q, version %q", azImage.imageDefinition, azImage.version)) poller, err := a.imageVersions.BeginDelete(ctx, azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, azImage.version, nil) if err != nil { return fmt.Errorf("begin delete image version: %w", err) } - log.Debugf("Waiting for operation to finish") + log.Debug("Waiting for operation to finish") if _, err := poller.PollUntilDone(ctx, nil); err != nil { return fmt.Errorf("waiting for operation: %w", err) } - log.Debugf("Checking if image definition %q still has versions left", azImage.imageDefinition) + log.Debug(fmt.Sprintf("Checking if image definition %q still has versions left", azImage.imageDefinition)) pager := a.imageVersions.NewListByGalleryImagePager(azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, nil) for pager.More() { @@ -656,20 +658,20 @@ func (a *azureClient) deleteImage(ctx context.Context, image string, dryrun bool return fmt.Errorf("listing image versions of image definition %s: %w", azImage.imageDefinition, err) } if len(nextResult.Value) != 0 { - log.Debugf("Image definition %q still has versions left, won't be deleted", azImage.imageDefinition) + log.Debug(fmt.Sprintf("Image definition %q still has versions left, won't be deleted", azImage.imageDefinition)) return nil } } time.Sleep(15 * time.Second) // Azure needs time understand that there is no version left... - log.Debugf("Deleting image definition %s", azImage.imageDefinition) + log.Debug(fmt.Sprintf("Deleting image definition %q", azImage.imageDefinition)) op, err := a.image.BeginDelete(ctx, azImage.resourceGroup, azImage.gallery, azImage.imageDefinition, nil) if err != nil { return fmt.Errorf("deleting image definition %s: %w", azImage.imageDefinition, err) } - log.Debugf("Waiting for operation to finish") + log.Debug("Waiting for operation to finish") if _, err := op.PollUntilDone(ctx, nil); err != nil { return fmt.Errorf("waiting for operation: %w", err) } @@ -684,12 +686,12 @@ type azImage struct { version string } -func (a *azureClient) parseImage(ctx context.Context, image string, log *logger.Logger) (azImage, error) { +func (a *azureClient) parseImage(ctx context.Context, image string, log *slog.Logger) (azImage, error) { if m := azImageRegex.FindStringSubmatch(image); len(m) == 5 { - log.Debugf( - "Image matches local image format, resource group: %s, gallery: %s, image definition: %s, version: %s", + log.Debug(fmt.Sprintf( + "Image matches local image format, resource group: %q, gallery: %q, image definition: %q, version: %q", m[1], m[2], m[3], m[4], - ) + )) return azImage{ resourceGroup: m[1], gallery: m[2], @@ -707,10 +709,10 @@ func (a *azureClient) parseImage(ctx context.Context, image string, log *logger. imageDefinition := m[2] version := m[3] - log.Debugf( - "Image matches community image format, gallery public name: %s, image definition: %s, version: %s", + log.Debug(fmt.Sprintf( + "Image matches community image format, gallery public name: %q, image definition: %q, version: %q", galleryPublicName, imageDefinition, version, - ) + )) var galleryName string pager := a.galleries.NewListPager(nil) @@ -721,24 +723,24 @@ func (a *azureClient) parseImage(ctx context.Context, image string, log *logger. } for _, v := range nextResult.Value { if v.Name == nil { - log.Debugf("Skipping gallery with nil name") + log.Debug("Skipping gallery with nil name") continue } if v.Properties.SharingProfile == nil { - log.Debugf("Skipping gallery %s with nil sharing profile", *v.Name) + log.Debug(fmt.Sprintf("Skipping gallery %q with nil sharing profile", *v.Name)) continue } if v.Properties.SharingProfile.CommunityGalleryInfo == nil { - log.Debugf("Skipping gallery %s with nil community gallery info", *v.Name) + log.Debug(fmt.Sprintf("Skipping gallery %q with nil community gallery info", *v.Name)) continue } if v.Properties.SharingProfile.CommunityGalleryInfo.PublicNames == nil { - log.Debugf("Skipping gallery %s with nil public names", *v.Name) + log.Debug(fmt.Sprintf("Skipping gallery %q with nil public names", *v.Name)) continue } for _, publicName := range v.Properties.SharingProfile.CommunityGalleryInfo.PublicNames { if publicName == nil { - log.Debugf("Skipping nil public name") + log.Debug("Skipping nil public name") continue } if *publicName == galleryPublicName { diff --git a/internal/api/versionsapi/client.go b/internal/api/versionsapi/client.go index 67900ec0e..295223d26 100644 --- a/internal/api/versionsapi/client.go +++ b/internal/api/versionsapi/client.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi @@ -10,29 +10,32 @@ import ( "context" "errors" "fmt" + "log/slog" "path" "golang.org/x/mod/semver" apiclient "github.com/edgelesssys/constellation/v2/internal/api/client" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" ) // Client is a client for the versions API. type Client struct { *apiclient.Client clientClose func(ctx context.Context) error + + log *slog.Logger } // NewClient creates a new client for the versions API. func NewClient(ctx context.Context, region, bucket, distributionID string, dryRun bool, - log *logger.Logger, + log *slog.Logger, ) (*Client, CloseFunc, error) { genericClient, genericClientClose, err := apiclient.NewClient(ctx, region, bucket, distributionID, dryRun, log) versionsClient := &Client{ - genericClient, - genericClientClose, + Client: genericClient, + clientClose: genericClientClose, + log: log, } versionsClientClose := func(ctx context.Context) error { return versionsClient.Close(ctx) @@ -43,15 +46,16 @@ func NewClient(ctx context.Context, region, bucket, distributionID string, dryRu // NewReadOnlyClient creates a new read-only client. // This client can be used to fetch objects but cannot write updates. func NewReadOnlyClient(ctx context.Context, region, bucket, distributionID string, - log *logger.Logger, + log *slog.Logger, ) (*Client, CloseFunc, error) { genericClient, genericClientClose, err := apiclient.NewReadOnlyClient(ctx, region, bucket, distributionID, log) if err != nil { return nil, nil, err } versionsClient := &Client{ - genericClient, - genericClientClose, + Client: genericClient, + clientClose: genericClientClose, + log: log, } versionsClientClose := func(ctx context.Context) error { return versionsClient.Close(ctx) @@ -131,18 +135,18 @@ func (c *Client) DeleteRef(ctx context.Context, ref string) error { func (c *Client) DeleteVersion(ctx context.Context, ver Version) error { var retErr error - c.Client.Logger.Debugf("Deleting version %s from minor version list", ver.version) + c.log.Debug(fmt.Sprintf("Deleting version %q from minor version list", ver.version)) possibleNewLatest, err := c.deleteVersionFromMinorVersionList(ctx, ver) if err != nil { retErr = errors.Join(retErr, fmt.Errorf("removing from minor version list: %w", err)) } - c.Client.Logger.Debugf("Checking latest version for %s", ver.version) + c.log.Debug(fmt.Sprintf("Checking latest version for %q", ver.version)) if err := c.deleteVersionFromLatest(ctx, ver, possibleNewLatest); err != nil { retErr = errors.Join(retErr, fmt.Errorf("updating latest version: %w", err)) } - c.Client.Logger.Debugf("Deleting artifact path %s for %s", ver.ArtifactPath(APIV1), ver.version) + c.log.Debug(fmt.Sprintf("Deleting artifact path %q for %q", ver.ArtifactPath(APIV1), ver.version)) if err := c.Client.DeletePath(ctx, ver.ArtifactPath(APIV1)); err != nil { retErr = errors.Join(retErr, fmt.Errorf("deleting artifact path: %w", err)) } @@ -159,20 +163,20 @@ func (c *Client) deleteVersionFromMinorVersionList(ctx context.Context, ver Vers Base: ver.WithGranularity(GranularityMinor), Kind: VersionKindImage, } - c.Client.Logger.Debugf("Fetching minor version list for version %s", ver.version) + c.log.Debug(fmt.Sprintf("Fetching minor version list for version %q", ver.version)) minorList, err := c.FetchVersionList(ctx, minorList) var notFoundErr *apiclient.NotFoundError if errors.As(err, ¬FoundErr) { - c.Client.Logger.Warnf("Minor version list for version %s not found", ver.version) - c.Client.Logger.Warnf("Skipping update of minor version list") + c.log.Warn(fmt.Sprintf("Minor version list for version %s not found", ver.version)) + c.log.Warn("Skipping update of minor version list") return nil, nil } else if err != nil { return nil, fmt.Errorf("fetching minor version list for version %s: %w", ver.version, err) } if !minorList.Contains(ver.version) { - c.Client.Logger.Warnf("Version %s is not in minor version list %s", ver.version, minorList.JSONPath()) - c.Client.Logger.Warnf("Skipping update of minor version list") + c.log.Warn(fmt.Sprintf("Version %s is not in minor version list %s", ver.version, minorList.JSONPath())) + c.log.Warn("Skipping update of minor version list") return nil, nil } @@ -192,20 +196,20 @@ func (c *Client) deleteVersionFromMinorVersionList(ctx context.Context, ver Vers Kind: VersionKindImage, Version: minorList.Versions[len(minorList.Versions)-1], } - c.Client.Logger.Debugf("Possible latest version replacement %q", latest.Version) + c.log.Debug(fmt.Sprintf("Possible latest version replacement %q", latest.Version)) } if c.Client.DryRun { - c.Client.Logger.Debugf("DryRun: Updating minor version list %s to %v", minorList.JSONPath(), minorList) + c.log.Debug(fmt.Sprintf("DryRun: Updating minor version list %q to %v", minorList.JSONPath(), minorList)) return latest, nil } - c.Client.Logger.Debugf("Updating minor version list %s", minorList.JSONPath()) + c.log.Debug(fmt.Sprintf("Updating minor version list %q", minorList.JSONPath())) if err := c.UpdateVersionList(ctx, minorList); err != nil { return latest, fmt.Errorf("updating minor version list %s: %w", minorList.JSONPath(), err) } - c.Client.Logger.Debugf("Removed version %s from minor version list %s", ver.version, minorList.JSONPath()) + c.log.Debug(fmt.Sprintf("Removed version %q from minor version list %q", ver.version, minorList.JSONPath())) return latest, nil } @@ -216,33 +220,33 @@ func (c *Client) deleteVersionFromLatest(ctx context.Context, ver Version, possi Stream: ver.stream, Kind: VersionKindImage, } - c.Client.Logger.Debugf("Fetching latest version from %s", latest.JSONPath()) + c.log.Debug(fmt.Sprintf("Fetching latest version from %q", latest.JSONPath())) latest, err := c.FetchVersionLatest(ctx, latest) var notFoundErr *apiclient.NotFoundError if errors.As(err, ¬FoundErr) { - c.Client.Logger.Warnf("Latest version for %s not found", latest.JSONPath()) + c.log.Warn(fmt.Sprintf("Latest version for %s not found", latest.JSONPath())) return nil } else if err != nil { return fmt.Errorf("fetching latest version: %w", err) } if latest.Version != ver.version { - c.Client.Logger.Debugf("Latest version is %s, not the deleted version %s", latest.Version, ver.version) + c.log.Debug(fmt.Sprintf("Latest version is %q, not the deleted version %q", latest.Version, ver.version)) return nil } if possibleNewLatest == nil { - c.Client.Logger.Errorf("Latest version is %s, but no new latest version was found", latest.Version) - c.Client.Logger.Errorf("A manual update of latest at %s might be needed", latest.JSONPath()) + c.log.Error(fmt.Sprintf("Latest version is %s, but no new latest version was found", latest.Version)) + c.log.Error(fmt.Sprintf("A manual update of latest at %s might be needed", latest.JSONPath())) return fmt.Errorf("latest version is %s, but no new latest version was found", latest.Version) } if c.Client.DryRun { - c.Client.Logger.Debugf("Would update latest version from %s to %s", latest.Version, possibleNewLatest.Version) + c.log.Debug(fmt.Sprintf("Would update latest version from %q to %q", latest.Version, possibleNewLatest.Version)) return nil } - c.Client.Logger.Infof("Updating latest version from %s to %s", latest.Version, possibleNewLatest.Version) + c.log.Info(fmt.Sprintf("Updating latest version from %s to %s", latest.Version, possibleNewLatest.Version)) if err := c.UpdateVersionLatest(ctx, *possibleNewLatest); err != nil { return fmt.Errorf("updating latest version: %w", err) } diff --git a/internal/api/versionsapi/cliinfo.go b/internal/api/versionsapi/cliinfo.go index ac7c18337..1ec6ac3d0 100644 --- a/internal/api/versionsapi/cliinfo.go +++ b/internal/api/versionsapi/cliinfo.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/cliinfo_test.go b/internal/api/versionsapi/cliinfo_test.go index f1f759f1b..2b85852b0 100644 --- a/internal/api/versionsapi/cliinfo_test.go +++ b/internal/api/versionsapi/cliinfo_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/fetcher.go b/internal/api/versionsapi/fetcher.go index e17d7a376..407fe1a70 100644 --- a/internal/api/versionsapi/fetcher.go +++ b/internal/api/versionsapi/fetcher.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/fetcher_test.go b/internal/api/versionsapi/fetcher_test.go index 810400af9..87245a9f8 100644 --- a/internal/api/versionsapi/fetcher_test.go +++ b/internal/api/versionsapi/fetcher_test.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi import ( "bytes" - "context" "encoding/json" "io" "net/http" @@ -21,7 +20,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestFetchVersionList(t *testing.T) { @@ -192,7 +191,7 @@ func TestFetchVersionList(t *testing.T) { fetcher := Fetcher{client, constants.CDNRepositoryURL} - list, err := fetcher.FetchVersionList(context.Background(), tc.list) + list, err := fetcher.FetchVersionList(t.Context(), tc.list) if tc.wantErr { assert.Error(err) diff --git a/internal/api/versionsapi/imageinfo.go b/internal/api/versionsapi/imageinfo.go index 59d01c1f2..77a14f666 100644 --- a/internal/api/versionsapi/imageinfo.go +++ b/internal/api/versionsapi/imageinfo.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/imageinfo_test.go b/internal/api/versionsapi/imageinfo_test.go index f239b42a2..04a72e941 100644 --- a/internal/api/versionsapi/imageinfo_test.go +++ b/internal/api/versionsapi/imageinfo_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/latest.go b/internal/api/versionsapi/latest.go index 2c9f2a20b..e360822fb 100644 --- a/internal/api/versionsapi/latest.go +++ b/internal/api/versionsapi/latest.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/latest_test.go b/internal/api/versionsapi/latest_test.go index a1b868d7b..562b5b4a3 100644 --- a/internal/api/versionsapi/latest_test.go +++ b/internal/api/versionsapi/latest_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/list.go b/internal/api/versionsapi/list.go index 9cef3ebaa..262007371 100644 --- a/internal/api/versionsapi/list.go +++ b/internal/api/versionsapi/list.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/list_test.go b/internal/api/versionsapi/list_test.go index 20aac9ab0..b936de898 100644 --- a/internal/api/versionsapi/list_test.go +++ b/internal/api/versionsapi/list_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi diff --git a/internal/api/versionsapi/version.go b/internal/api/versionsapi/version.go index 12d1e8100..0c969e0f6 100644 --- a/internal/api/versionsapi/version.go +++ b/internal/api/versionsapi/version.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi @@ -41,7 +41,7 @@ type Version struct { // NewVersion creates a new Version object and validates it. func NewVersion(ref, stream, version string, kind VersionKind) (Version, error) { ver := Version{ - ref: ref, + ref: CanonicalizeRef(ref), stream: stream, version: version, kind: kind, @@ -62,7 +62,7 @@ func NewVersionFromShortPath(shortPath string, kind VersionKind) (Version, error } ver := Version{ - ref: ref, + ref: ref, // Canonicalized by parseShortPath. stream: stream, version: version, kind: kind, @@ -331,7 +331,7 @@ func CanonicalizeRef(ref string) string { canRef := notAZ09Regexp.ReplaceAllString(ref, "-") if canRef == ReleaseRef { - return "" // No ref should be cannonicalized to the release ref. + return "" // No ref should be canonicalized to the release ref. } return canRef @@ -401,7 +401,7 @@ func MeasurementURL(version Version) (measurementURL, signatureURL *url.URL, err } var ( - shortPathRegex = regexp.MustCompile(`^ref/([a-zA-Z0-9-]+)/stream/([a-zA-Z0-9-]+)/([a-zA-Z0-9.-]+)$`) + shortPathRegex = regexp.MustCompile(`^ref/([^/]+)/stream/([a-zA-Z0-9-]+)/([a-zA-Z0-9.-]+)$`) shortPathReleaseRegex = regexp.MustCompile(`^stream/([a-zA-Z0-9-]+)/([a-zA-Z0-9.-]+)$`) ) @@ -422,6 +422,7 @@ func parseShortPath(shortPath string) (ref, stream, version string, err error) { if shortPathRegex.MatchString(shortPath) { matches := shortPathRegex.FindStringSubmatch(shortPath) ref := matches[1] + ref = CanonicalizeRef(ref) if err := ValidateRef(ref); err != nil { return "", "", "", err } diff --git a/internal/api/versionsapi/version_test.go b/internal/api/versionsapi/version_test.go index 25f0f8ce0..c3631c90c 100644 --- a/internal/api/versionsapi/version_test.go +++ b/internal/api/versionsapi/version_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package versionsapi @@ -16,6 +16,111 @@ import ( "github.com/edgelesssys/constellation/v2/internal/constants" ) +func TestNewVersion(t *testing.T) { + testCases := map[string]struct { + ref string + stream string + version string + kind VersionKind + wantVer Version + wantErr bool + }{ + "stable release image": { + ref: ReleaseRef, + stream: "stable", + version: "v9.9.9", + kind: VersionKindImage, + wantVer: Version{ + ref: ReleaseRef, + stream: "stable", + version: "v9.9.9", + kind: VersionKindImage, + }, + }, + "release debug image": { + ref: ReleaseRef, + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + wantVer: Version{ + ref: ReleaseRef, + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + }, + }, + "stable release cli": { + ref: ReleaseRef, + stream: "stable", + version: "v9.9.9", + kind: VersionKindCLI, + wantVer: Version{ + ref: ReleaseRef, + stream: "stable", + version: "v9.9.9", + kind: VersionKindCLI, + }, + }, + "release debug cli": { + ref: ReleaseRef, + stream: "debug", + version: "v9.9.9", + kind: VersionKindCLI, + wantVer: Version{ + ref: ReleaseRef, + stream: "debug", + version: "v9.9.9", + kind: VersionKindCLI, + }, + }, + "unknown kind": { + ref: ReleaseRef, + stream: "debug", + version: "v9.9.9", + kind: VersionKindUnknown, + wantErr: true, + }, + "non-release ref as input": { + ref: "working-branch", + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + wantVer: Version{ + ref: "working-branch", + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + }, + }, + "non-canonical ref as input": { + ref: "testing-1.23", + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + wantVer: Version{ + ref: "testing-1-23", + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + ver, err := NewVersion(tc.ref, tc.stream, tc.version, tc.kind) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + assert.Equal(tc.wantVer, ver) + }) + } +} + func TestNewVersionFromShortPath(t *testing.T) { testCases := map[string]struct { path string @@ -78,6 +183,26 @@ func TestNewVersionFromShortPath(t *testing.T) { kind: VersionKindCLI, wantErr: true, }, + "non-release ref as input": { + path: "ref/working-branch/stream/debug/v9.9.9", + kind: VersionKindImage, + wantVer: Version{ + ref: "working-branch", + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + }, + }, + "non-canonical ref as input": { + path: "ref/testing-1.23/stream/debug/v9.9.9", + kind: VersionKindImage, + wantVer: Version{ + ref: "testing-1-23", + stream: "debug", + version: "v9.9.9", + kind: VersionKindImage, + }, + }, } for name, tc := range testCases { diff --git a/internal/api/versionsapi/versionsapi.go b/internal/api/versionsapi/versionsapi.go index 54fa65123..b89cd7577 100644 --- a/internal/api/versionsapi/versionsapi.go +++ b/internal/api/versionsapi/versionsapi.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/atls/atls.go b/internal/atls/atls.go index 06fc38cb0..9f42fb9c9 100644 --- a/internal/atls/atls.go +++ b/internal/atls/atls.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // aTLS provides config generation functions to bootstrap attested TLS connections. @@ -70,6 +70,7 @@ func CreateAttestationClientTLSConfig(issuer Issuer, validators []Validator) (*t InsecureSkipVerify: true, // disable default verification because we use our own verify func ServerName: base64.StdEncoding.EncodeToString(clientNonce), // abuse ServerName as a channel to transmit the nonce MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1", "h2"}, // grpc-go requires us to advertise HTTP/2 (h2) over ALPN }, nil } @@ -96,7 +97,7 @@ func getATLSConfigForClientFunc(issuer Issuer, validators []Validator) (func(*tl } // this function will be called once for every client - return func(chi *tls.ClientHelloInfo) (*tls.Config, error) { + return func(_ *tls.ClientHelloInfo) (*tls.Config, error) { // generate nonce for this connection serverNonce, err := crypto.GenerateRandomBytes(crypto.RNGLengthDefault) if err != nil { @@ -114,6 +115,7 @@ func getATLSConfigForClientFunc(issuer Issuer, validators []Validator) (func(*tl VerifyPeerCertificate: serverConn.verify, GetCertificate: serverConn.getCertificate, MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1", "h2"}, // grpc-go requires us to advertise HTTP/2 (h2) over ALPN } // enable mutual aTLS if any validators are set @@ -200,6 +202,7 @@ func processCertificate(rawCerts [][]byte, _ [][]*x509.Certificate) (*x509.Certi // verifyEmbeddedReport verifies an aTLS certificate by validating the attestation document embedded in the TLS certificate. func verifyEmbeddedReport(validators []Validator, cert *x509.Certificate, hash, nonce []byte) error { + var exts []string for _, ex := range cert.Extensions { for _, validator := range validators { if ex.Id.Equal(validator.OID()) { @@ -216,9 +219,10 @@ func verifyEmbeddedReport(validators []Validator, cert *x509.Certificate, hash, return nil } } + exts = append(exts, ex.Id.String()) } - return errors.New("certificate does not contain attestation document") + return fmt.Errorf("certificate does not contain compatible attestation documents: got extension OIDs %#v", exts) } func hashPublicKey(pub any) ([]byte, error) { diff --git a/internal/atls/atls_test.go b/internal/atls/atls_test.go index 063304fcf..03f1660c6 100644 --- a/internal/atls/atls_test.go +++ b/internal/atls/atls_test.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package atls import ( - "context" "encoding/asn1" "errors" "io" @@ -22,7 +21,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestTLSConfig(t *testing.T) { @@ -142,7 +141,7 @@ func TestTLSConfig(t *testing.T) { serverConfig, err := CreateAttestationServerTLSConfig(tc.serverIssuer, tc.serverValidators) require.NoError(err) - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "hello") })) server.TLS = serverConfig @@ -162,7 +161,7 @@ func TestTLSConfig(t *testing.T) { server.StartTLS() defer server.Close() - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, http.NoBody) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, http.NoBody) require.NoError(err) resp, err := client.Do(req) if tc.wantErr { @@ -196,7 +195,7 @@ func TestClientConnectionConcurrency(t *testing.T) { serverCfg, err := CreateAttestationServerTLSConfig(NewFakeIssuer(variant.Dummy{}), NewFakeValidators(variant.Dummy{})) require.NoError(err) - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "hello") })) server.TLS = serverCfg @@ -221,7 +220,7 @@ func TestClientConnectionConcurrency(t *testing.T) { var reqs []*http.Request for _, url := range urls { - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, http.NoBody) require.NoError(err) reqs = append(reqs, req) } @@ -270,7 +269,7 @@ func TestServerConnectionConcurrency(t *testing.T) { require.NoError(err) for i := 0; i < serverCount; i++ { - server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "hello") })) server.TLS = serverCfg @@ -295,7 +294,7 @@ func TestServerConnectionConcurrency(t *testing.T) { var reqs []*http.Request for _, url := range urls { - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, http.NoBody) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, http.NoBody) require.NoError(err) reqs = append(reqs, req) } diff --git a/internal/attestation/attestation.go b/internal/attestation/attestation.go index f09988dce..21b918925 100644 --- a/internal/attestation/attestation.go +++ b/internal/attestation/attestation.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -45,18 +45,18 @@ const ( // Logger is a logger used to print warnings and infos during attestation validation. type Logger interface { - Infof(format string, args ...any) - Warnf(format string, args ...any) + Info(msg string, args ...any) + Warn(msg string, args ...any) } // NOPLogger is a no-op implementation of [Logger]. type NOPLogger struct{} -// Infof is a no-op. -func (NOPLogger) Infof(string, ...interface{}) {} +// Info is a no-op. +func (NOPLogger) Info(string, ...interface{}) {} -// Warnf is a no-op. -func (NOPLogger) Warnf(string, ...interface{}) {} +// Warn is a no-op. +func (NOPLogger) Warn(string, ...interface{}) {} // DeriveClusterID derives the cluster ID from a salt and secret value. func DeriveClusterID(secret, salt []byte) ([]byte, error) { diff --git a/internal/attestation/attestation_test.go b/internal/attestation/attestation_test.go index 3615859c0..482089dee 100644 --- a/internal/attestation/attestation_test.go +++ b/internal/attestation/attestation_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package attestation diff --git a/internal/attestation/aws/aws.go b/internal/attestation/aws/aws.go index 80806eccd..5edaaea50 100644 --- a/internal/attestation/aws/aws.go +++ b/internal/attestation/aws/aws.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/aws/nitrotpm/issuer.go b/internal/attestation/aws/nitrotpm/issuer.go index e95b72d06..5365eed82 100644 --- a/internal/attestation/aws/nitrotpm/issuer.go +++ b/internal/attestation/aws/nitrotpm/issuer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package nitrotpm diff --git a/internal/attestation/aws/nitrotpm/issuer_test.go b/internal/attestation/aws/nitrotpm/issuer_test.go index 59b5b7e47..eb2733775 100644 --- a/internal/attestation/aws/nitrotpm/issuer_test.go +++ b/internal/attestation/aws/nitrotpm/issuer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package nitrotpm @@ -100,7 +100,7 @@ func TestGetInstanceInfo(t *testing.T) { instanceInfoFunc := getInstanceInfo(&tc.client) assert.NotNil(instanceInfoFunc) - info, err := instanceInfoFunc(context.Background(), tpm, nil) + info, err := instanceInfoFunc(t.Context(), tpm, nil) if tc.wantErr { assert.Error(err) assert.Nil(info) diff --git a/internal/attestation/aws/nitrotpm/nitrotpm.go b/internal/attestation/aws/nitrotpm/nitrotpm.go index 47c59db9b..a86c417fe 100644 --- a/internal/attestation/aws/nitrotpm/nitrotpm.go +++ b/internal/attestation/aws/nitrotpm/nitrotpm.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/aws/nitrotpm/validator.go b/internal/attestation/aws/nitrotpm/validator.go index 54d853c5d..ea60e1685 100644 --- a/internal/attestation/aws/nitrotpm/validator.go +++ b/internal/attestation/aws/nitrotpm/validator.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package nitrotpm diff --git a/internal/attestation/aws/nitrotpm/validator_test.go b/internal/attestation/aws/nitrotpm/validator_test.go index 0e6d086cd..a782894fa 100644 --- a/internal/attestation/aws/nitrotpm/validator_test.go +++ b/internal/attestation/aws/nitrotpm/validator_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package nitrotpm @@ -42,7 +42,7 @@ func TestGeTrustedKey(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) out, err := getTrustedKey( - context.Background(), + t.Context(), vtpm.AttestationDocument{ Attestation: &attest.Attestation{ AkPub: tc.akPub, diff --git a/internal/attestation/aws/snp/BUILD.bazel b/internal/attestation/aws/snp/BUILD.bazel index f08964307..f8287da48 100644 --- a/internal/attestation/aws/snp/BUILD.bazel +++ b/internal/attestation/aws/snp/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//internal/attestation/vtpm", "//internal/config", "@com_github_google_go_sev_guest//abi", - "@com_github_google_go_sev_guest//client", "@com_github_google_go_sev_guest//kds", "@com_github_google_go_sev_guest//proto/sevsnp", "@com_github_google_go_sev_guest//validate", diff --git a/internal/attestation/aws/snp/errors.go b/internal/attestation/aws/snp/errors.go index 2b07870b7..b20291b5e 100644 --- a/internal/attestation/aws/snp/errors.go +++ b/internal/attestation/aws/snp/errors.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp diff --git a/internal/attestation/aws/snp/issuer.go b/internal/attestation/aws/snp/issuer.go index e3d58ab79..040a19a94 100644 --- a/internal/attestation/aws/snp/issuer.go +++ b/internal/attestation/aws/snp/issuer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -21,7 +21,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" "github.com/google/go-sev-guest/abi" - sevclient "github.com/google/go-sev-guest/client" "github.com/google/go-tpm-tools/client" tpmclient "github.com/google/go-tpm-tools/client" ) @@ -70,13 +69,7 @@ func getInstanceInfo(_ context.Context, tpm io.ReadWriteCloser, _ []byte) ([]byt akDigest := sha512.Sum512(encoded) - device, err := sevclient.OpenDevice() - if err != nil { - return nil, fmt.Errorf("opening sev device: %w", err) - } - defer device.Close() - - report, certs, err := sevclient.GetRawExtendedReportAtVmpl(device, akDigest, 0) + report, certs, err := snp.GetExtendedReport(akDigest) if err != nil { return nil, fmt.Errorf("getting extended report: %w", err) } diff --git a/internal/attestation/aws/snp/issuer_test.go b/internal/attestation/aws/snp/issuer_test.go index 3f2f24699..d92cad9c9 100644 --- a/internal/attestation/aws/snp/issuer_test.go +++ b/internal/attestation/aws/snp/issuer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp diff --git a/internal/attestation/aws/snp/snp.go b/internal/attestation/aws/snp/snp.go index 4085f50bb..4b5f31d6a 100644 --- a/internal/attestation/aws/snp/snp.go +++ b/internal/attestation/aws/snp/snp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/aws/snp/testdata/report.txt b/internal/attestation/aws/snp/testdata/report.txt index efd90375f..e413ca309 100644 --- a/internal/attestation/aws/snp/testdata/report.txt +++ b/internal/attestation/aws/snp/testdata/report.txt @@ -1 +1 @@ -AgAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAADAAAAAAAK0QMAAAAAAAAABAAAAAAAAAADJjVhPI4zH6KeCWNxkQ/mofaTg92gLJRhQApwtm2Ho9pd2GMAJSK+Q6/DTywjOYm9bkAeNR0Q18yADW9d/PAZJayBD1xHUIkPsaFY8JeWLgTU1/tkDR0IqZgpz0pwVDpHzG+xkrvpCqcTFCNhpmFVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACOsAob9aWVVnjx8VNbU/bqGewnLGnBSZbJu8smGfzcN///////////////////////////////////////////AwAAAAAACnMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAACqkBNgEAATYBAAMAAAAAAAqpAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAm8z1/Oxcd+Bhdxd1okDoZ9gMiYw5Y/fp74hylcA2Eu+XPt5p+7fqqG7d7YLdJtTuAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIOZBrwmRpIFfKDCywiFaiILyguTq/6vefDmdzNBKiRKtjdNiHa0hNgeQFGHspRcZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= +AwAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAEAAAAAAAY3CcAAAAAAAAABAAAAAAAAACHq3yvUQ4bNSDcPM62TuRBKOEJdvsNP8XidGmdiq9QYVSvTB3goCa0n9+GHprHVVFVGzU00cYTaaOwj1uu0NsvWlzbrY9UDOSrygEg+uyG9i73EopKxxGt001Gi1lXyCqi8k6PER2tw2ibeHuI7QcVAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARowRHpiQyfxJKbRS+DVfQGWxQvKf1S21qaW2zACl7rf//////////////////////////////////////////BAAAAAAAGNkZAQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAGNsdNwEAHTcBAAQAAAAAABjbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATEij8MQ3cc95xvjozFQCY/3yYhrUJa6qN5kOaH0eHbuMzQ0iOgY3m6riTBYsQlksAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAt0MH84001UcDpwNKn6LJSVfidlQxQ2nAM6WGsDjMvA4Z8WcYJeQhgpcDL7YJ+dbpAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= diff --git a/internal/attestation/aws/snp/testdata/testdata.go b/internal/attestation/aws/snp/testdata/testdata.go index 61d14f154..504693be7 100644 --- a/internal/attestation/aws/snp/testdata/testdata.go +++ b/internal/attestation/aws/snp/testdata/testdata.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package testdata contains testing data for an attestation process. @@ -15,7 +15,7 @@ import _ "embed" var SNPReport string // AKDigest holds the AK digest embedded in SNPReport.REPORT_DATA. -const AKDigest = "032635613c8e331fa29e096371910fe6a1f69383dda02c9461400a70b66d87a3da5dd863002522be43afc34f2c233989bd6e401e351d10d7cc800d6f5dfcf019" +const AKDigest = "87ab7caf510e1b3520dc3cceb64ee44128e10976fb0d3fc5e274699d8aaf506154af4c1de0a026b49fdf861e9ac75551551b3534d1c61369a3b08f5baed0db2f" // VLEK for SNPReport. // diff --git a/internal/attestation/aws/snp/testdata/vlek.pem b/internal/attestation/aws/snp/testdata/vlek.pem index 406a84235..3f8be0331 100644 --- a/internal/attestation/aws/snp/testdata/vlek.pem +++ b/internal/attestation/aws/snp/testdata/vlek.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFLDCCAtugAwIBAgIBADBGBgkqhkiG9w0BAQowOaAPMA0GCWCGSAFlAwQCAgUA -oRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAgUAogMCATCjAwIBATCBgDEUMBIG -A1UECwwLRW5naW5lZXJpbmcxCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtTYW50YSBD -bGFyYTELMAkGA1UECAwCQ0ExHzAdBgNVBAoMFkFkdmFuY2VkIE1pY3JvIERldmlj -ZXMxFzAVBgNVBAMMDlNFVi1WTEVLLU1pbGFuMB4XDTIzMDcxOTA4MjkyOFoXDTI0 -MDcxOTA4MjkyOFowejEUMBIGA1UECwwLRW5naW5lZXJpbmcxCzAJBgNVBAYTAlVT -MRQwEgYDVQQHDAtTYW50YSBDbGFyYTELMAkGA1UECAwCQ0ExHzAdBgNVBAoMFkFk -dmFuY2VkIE1pY3JvIERldmljZXMxETAPBgNVBAMMCFNFVi1WTEVLMHYwEAYHKoZI -zj0CAQYFK4EEACIDYgAEXFl4NHpiQCuZXIrehIEk/5XNIdMvo24wyaezN+0FouYB -9Z23nL523gpJUlT+mvb5ZMybh5tO1nBGFMOKwzP9dnSBwTs0qn57Ts9OTpW57EAo -Mx4SI7g1yz/mt4e6hma4o4HxMIHuMBAGCSsGAQQBnHgBAQQDAgEAMBQGCSsGAQQB -nHgBAgQHFgVNaWxhbjARBgorBgEEAZx4AQMBBAMCAQMwEQYKKwYBBAGceAEDAgQD -AgEAMBEGCisGAQQBnHgBAwQEAwIBADARBgorBgEEAZx4AQMFBAMCAQAwEQYKKwYB -BAGceAEDBgQDAgEAMBEGCisGAQQBnHgBAwcEAwIBADARBgorBgEEAZx4AQMDBAMC -AQowEQYKKwYBBAGceAEDCAQDAgFzMCwGCSsGAQQBnHgBBQQfFh1DTj1jYy11cy1l -YXN0LTIuYW1hem9uYXdzLmNvbTBGBgkqhkiG9w0BAQowOaAPMA0GCWCGSAFlAwQC -AgUAoRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAgUAogMCATCjAwIBAQOCAgEA -E2CR10QkVTofcjmQbuu787J+H+OjzQLPIi/dUbP/LvZdYi/eWglYQPRbYxhxnIi1 -PB9R9c7LLhbNRhroog+TzrxyKLibEAW3rwn2iygPnsIemyL89wqtPNqEKNjhBXsb -s/0bmf0rNJ3lugssCAzrIStkx8at0K/099BEs4FuUM5u97HVy+jqLdRa2XOHMgGa -K7sNdR4swuLhfts9gOOX8ntJ+XkxtUx2mz449fXn8KN70mKa2YShhNd2JWJmv1jW -K0I1UxVVwIOHBn/W8fQL5a061oRQQaW5+wPRTys0iEMmLU7+plC8LNWeEq93TfFY -eUZ9EzinZ5S7z+c8J1FVWYNHGJauWj4lkjf+XGUZqXwTCPzou6tYJqqwWQEUUxXC -M3QKgbkIGWg4WKHIAXGChbM86JLY0W6VueOHyu4S1Z4i81IcDp4cs83WxYWfCpKH -Fq3Si2BhzZ0YGgK25JCkomh5Yf7dlsByyuQssf3TCqNmOfSFOTLvxfwTvLD5Omlm -O1mPI0YaoZya4WcPxbpWS+2Em23/5inQvT+ZhvMNkljD2NVbhLVGP1v4YR+T2zaC -0qJ4YYJ2ERQTnEUlKnlF9bm6PwZSRHupK6ecsGjH+Bz5hBPbT09nEpJf0bWkzVSA -AY8POFt3zBJiqONQuOlBpXzqKRKvFYQVEaX2EXQ+W6s= +MIIFIzCCAtegAwIBAgIBADBBBgkqhkiG9w0BAQowNKAPMA0GCWCGSAFlAwQCAgUA +oRwwGgYJKoZIhvcNAQEIMA0GCWCGSAFlAwQCAgUAogMCATAwgYAxFDASBgNVBAsM +C0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEgQ2xhcmEx +CzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZpY2VzMRcw +FQYDVQQDDA5TRVYtVkxFSy1NaWxhbjAeFw0yNDEyMTAyMjMwMTZaFw0yNTEyMTAy +MjMwMTZaMHoxFDASBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIG +A1UEBwwLU2FudGEgQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNl +ZCBNaWNybyBEZXZpY2VzMREwDwYDVQQDDAhTRVYtVkxFSzB2MBAGByqGSM49AgEG +BSuBBAAiA2IABJRw6hwLZt7KX95uPePz/3Gt/z9mm/32f0JpE2twW8w6DQ1xOPnW +YRLJeMSZNpaYW/NRpNf0vfy5IDQt44didvu+37x2aqyaneFiBh5jTxSg/2dCZ+bi +4eZw/p0Us7bubqOB8jCB7zAQBgkrBgEEAZx4AQEEAwIBADAUBgkrBgEEAZx4AQIE +BxYFTWlsYW4wEQYKKwYBBAGceAEDAQQDAgEEMBEGCisGAQQBnHgBAwIEAwIBADAR +BgorBgEEAZx4AQMEBAMCAQAwEQYKKwYBBAGceAEDBQQDAgEAMBEGCisGAQQBnHgB +AwYEAwIBADARBgorBgEEAZx4AQMHBAMCAQAwEQYKKwYBBAGceAEDAwQDAgEYMBIG +CisGAQQBnHgBAwgEBAICANkwLAYJKwYBBAGceAEFBB8WHUNOPWNjLWV1LXdlc3Qt +MS5hbWF6b25hd3MuY29tMEEGCSqGSIb3DQEBCjA0oA8wDQYJYIZIAWUDBAICBQCh +HDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMAOCAgEAar1tA7vYelxK +uj+r7APOEPcAAoF7RWZs6ixDlXHuFVj2rfxqmxt8nqjedEKBfUGPCEsbAV+Z/bj9 +GqN+q5Bn1yk6RL/VqxTxTVhpa0G33R87UjE+S+42k6ENgddbl4hxws5g83Sn9All +/XjNPHmciWjmix4PJs5tZv+YaJ15BSBkJfrTRo+rX3UDKeqUHNoX+Cx6D7ECF/6k +ToFlHBEBqHKa2EzhNMK2UXm/vm0ATSaNHuDEGBvzbXflPmHZi1RZqQ7q9VMenFDx +JwAgnUrltcuLjKMID7c2yj+Emk/CBEEFoAJRfSoSvMdhqrNaRlbEqEnQ95C/XNPn +Mqtk5Ao/UVV5fRXYSt5oGKTBGhqTwv+Xqyei+/IgpcJyGPFbHVX9UPteP4RnSLiq +uJ3oRIvyEw+u6bkMNBBAjh4C+Jp2BVrLs1aC0h9fjfVEofWTb/NioJRigKTNfbao +sTy6tX8qoUSxtp/bIqK1jg1Y7eIDIMCgqnm0N+hJT7CnkwyCBUkOHmsExzQcthmg +y0J1J7bTA507rY5ZglNSRLCXqAfORVxIBwTaOXrJV2lMLScTUdnhFrVPFUAl7uCj +rKta1iGye+fieoYncdHLIVyIJGsTC+AbhPIAR2Zh847Sxw1SVOobTPc0wUIoKrOU +xR32EkufsNGLb8TiEsgpa2ulbw8xi6U= -----END CERTIFICATE----- diff --git a/internal/attestation/aws/snp/validator.go b/internal/attestation/aws/snp/validator.go index 22d8b814b..92431929f 100644 --- a/internal/attestation/aws/snp/validator.go +++ b/internal/attestation/aws/snp/validator.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -31,7 +31,7 @@ import ( // Validator for AWS TPM attestation. type Validator struct { - // Embed variant to identify the Validator using varaint.OID(). + // Embed variant to identify the Validator using variant.OID(). variant.AWSSEVSNP // Embed validator to implement Validate method for aTLS handshake. *vtpm.Validator @@ -191,11 +191,11 @@ func (a *awsValidator) validate(attestation vtpm.AttestationDocument, ask *x509. func getVerifyOpts(att *sevsnp.Attestation) (*verify.Options, error) { ask, err := x509.ParseCertificate(att.CertificateChain.AskCert) if err != nil { - return &verify.Options{}, fmt.Errorf("parsing VLEK certificate: %w", err) + return nil, fmt.Errorf("parsing ASK certificate: %w", err) } ark, err := x509.ParseCertificate(att.CertificateChain.ArkCert) if err != nil { - return &verify.Options{}, fmt.Errorf("parsing VLEK certificate: %w", err) + return nil, fmt.Errorf("parsing ARK certificate: %w", err) } verifyOpts := &verify.Options{ diff --git a/internal/attestation/aws/snp/validator_test.go b/internal/attestation/aws/snp/validator_test.go index 84804a886..567791daf 100644 --- a/internal/attestation/aws/snp/validator_test.go +++ b/internal/attestation/aws/snp/validator_test.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp import ( "bytes" - "context" "crypto" "crypto/x509" "encoding/base64" @@ -67,7 +66,7 @@ func TestGetTrustedKey(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) out, err := validator().getTrustedKey( - context.Background(), + t.Context(), vtpm.AttestationDocument{ Attestation: &attest.Attestation{ AkPub: tc.akPub, diff --git a/internal/attestation/azure/BUILD.bazel b/internal/attestation/azure/BUILD.bazel index 93cc51ed5..57f5d242b 100644 --- a/internal/attestation/azure/BUILD.bazel +++ b/internal/attestation/azure/BUILD.bazel @@ -1,8 +1,27 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") go_library( name = "azure", srcs = ["azure.go"], importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/azure", visibility = ["//:__subpackages__"], + deps = [ + "@com_github_google_go_tpm//legacy/tpm2", + "@com_github_google_go_tpm_tools//client", + ], +) + +go_test( + name = "azure_test", + srcs = ["azure_test.go"], + embed = [":azure"], + deps = [ + "//internal/attestation/simulator", + "//internal/attestation/snp", + "@com_github_google_go_tpm//legacy/tpm2", + "@com_github_google_go_tpm_tools//client", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], ) diff --git a/internal/attestation/azure/azure.go b/internal/attestation/azure/azure.go index 53e6a9466..655be1b2c 100644 --- a/internal/attestation/azure/azure.go +++ b/internal/attestation/azure/azure.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -13,8 +13,104 @@ Constellation supports multiple attestation technologies on Azure. TPM attestation verified using an SEV-SNP attestation statement. + - TDX - Trust Domain Extensions + + TPM attestation verified using a TDX attestation statement. + - Trusted Launch Basic TPM attestation. */ package azure + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + + tpmclient "github.com/google/go-tpm-tools/client" + "github.com/google/go-tpm/legacy/tpm2" +) + +const ( + // tpmAkIdx is the NV index of the attestation key used by Azure VMs. + tpmAkIdx = 0x81000003 +) + +// GetAttestationKey reads the attestation key put into the TPM during early boot. +func GetAttestationKey(tpm io.ReadWriter) (*tpmclient.Key, error) { + ak, err := tpmclient.LoadCachedKey(tpm, tpmAkIdx, tpmclient.NullSession{}) + if err != nil { + return nil, fmt.Errorf("reading HCL attestation key from TPM: %w", err) + } + + return ak, nil +} + +// HCLAkValidator validates an attestation key issued by the Host Compatibility Layer (HCL). +// The HCL is written by Azure, and sits between the Hypervisor and CVM OS. +// The HCL runs in the protected context of the CVM. +type HCLAkValidator struct{} + +// Validate validates that the attestation key from the TPM is trustworthy. The steps are: +// 1. runtime data read from the TPM has the same sha256 digest as reported in `report_data` of the SNP report or `TdQuoteBody.ReportData` of the TDX report. +// 2. modulus reported in runtime data matches modulus from key at idx 0x81000003. +// 3. exponent reported in runtime data matches exponent from key at idx 0x81000003. +// The function is currently tested manually on a Azure Ubuntu CVM. +func (a *HCLAkValidator) Validate(runtimeDataRaw []byte, reportData []byte, rsaParameters *tpm2.RSAParams) error { + var rtData runtimeData + if err := json.Unmarshal(runtimeDataRaw, &rtData); err != nil { + return fmt.Errorf("unmarshalling json: %w", err) + } + + sum := sha256.Sum256(runtimeDataRaw) + if len(reportData) < len(sum) { + return fmt.Errorf("reportData has unexpected size: %d", len(reportData)) + } + if !bytes.Equal(sum[:], reportData[:len(sum)]) { + return errors.New("unexpected runtimeData digest in TPM") + } + + if len(rtData.PublicPart) < 1 { + return errors.New("did not receive any keys in runtime data") + } + rawN, err := base64.RawURLEncoding.DecodeString(rtData.PublicPart[0].N) + if err != nil { + return fmt.Errorf("decoding modulus string: %w", err) + } + if !bytes.Equal(rawN, rsaParameters.ModulusRaw) { + return fmt.Errorf("unexpected modulus value in TPM") + } + + rawE, err := base64.RawURLEncoding.DecodeString(rtData.PublicPart[0].E) + if err != nil { + return fmt.Errorf("decoding exponent string: %w", err) + } + paddedRawE := make([]byte, 4) + copy(paddedRawE, rawE) + exponent := binary.LittleEndian.Uint32(paddedRawE) + + // According to this comment [1] the TPM uses "0" to represent the default exponent "65537". + // The go tpm library also reports the exponent as 0. Thus we have to handle it specially. + // [1] https://github.com/tpm2-software/tpm2-tools/pull/1973#issue-596685005 + if !((exponent == 65537 && rsaParameters.ExponentRaw == 0) || exponent == rsaParameters.ExponentRaw) { + return fmt.Errorf("unexpected N value in TPM") + } + + return nil +} + +type runtimeData struct { + PublicPart []akPub `json:"keys"` +} + +// akPub are the public parameters of an RSA attestation key. +type akPub struct { + E string `json:"e"` + N string `json:"n"` +} diff --git a/internal/attestation/azure/azure_test.go b/internal/attestation/azure/azure_test.go new file mode 100644 index 000000000..d73db409e --- /dev/null +++ b/internal/attestation/azure/azure_test.go @@ -0,0 +1,165 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package azure + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "os" + "testing" + + "github.com/edgelesssys/constellation/v2/internal/attestation/simulator" + "github.com/edgelesssys/constellation/v2/internal/attestation/snp" + "github.com/google/go-tpm-tools/client" + tpmclient "github.com/google/go-tpm-tools/client" + "github.com/google/go-tpm/legacy/tpm2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestValidateAk tests the attestation key validation with a simulated TPM device. +func TestValidateAk(t *testing.T) { + cgo := os.Getenv("CGO_ENABLED") + if cgo == "0" { + t.Skip("skipping test because CGO is disabled and tpm simulator requires it") + } + + int32ToBytes := func(val uint32) []byte { + r := make([]byte, 4) + binary.PutUvarint(r, uint64(val)) + return r + } + + require := require.New(t) + + tpm, err := simulator.OpenSimulatedTPM() + require.NoError(err) + defer tpm.Close() + key, err := client.AttestationKeyRSA(tpm) + require.NoError(err) + defer key.Close() + + e := base64.RawURLEncoding.EncodeToString(int32ToBytes(key.PublicArea().RSAParameters.ExponentRaw)) + n := base64.RawURLEncoding.EncodeToString(key.PublicArea().RSAParameters.ModulusRaw) + ak := akPub{E: e, N: n} + rtData := runtimeData{PublicPart: []akPub{ak}} + + defaultRuntimeDataRaw, err := json.Marshal(rtData) + require.NoError(err) + defaultInstanceInfo := snp.InstanceInfo{Azure: &snp.AzureInstanceInfo{RuntimeData: defaultRuntimeDataRaw}} + + sig := sha256.Sum256(defaultRuntimeDataRaw) + defaultReportData := sig[:] + defaultRsaParams := key.PublicArea().RSAParameters + + testCases := map[string]struct { + instanceInfo snp.InstanceInfo + runtimeDataRaw []byte + reportData []byte + rsaParameters *tpm2.RSAParams + wantErr bool + }{ + "success": { + instanceInfo: defaultInstanceInfo, + runtimeDataRaw: defaultRuntimeDataRaw, + reportData: defaultReportData, + rsaParameters: defaultRsaParams, + }, + "invalid json": { + instanceInfo: defaultInstanceInfo, + runtimeDataRaw: []byte(""), + reportData: defaultReportData, + rsaParameters: defaultRsaParams, + wantErr: true, + }, + "invalid hash": { + instanceInfo: defaultInstanceInfo, + runtimeDataRaw: defaultRuntimeDataRaw, + reportData: bytes.Repeat([]byte{0}, 64), + rsaParameters: defaultRsaParams, + wantErr: true, + }, + "invalid E": { + instanceInfo: defaultInstanceInfo, + runtimeDataRaw: defaultRuntimeDataRaw, + reportData: defaultReportData, + rsaParameters: func() *tpm2.RSAParams { + tmp := *defaultRsaParams + tmp.ExponentRaw = 1 + return &tmp + }(), + wantErr: true, + }, + "invalid N": { + instanceInfo: defaultInstanceInfo, + runtimeDataRaw: defaultRuntimeDataRaw, + reportData: defaultReportData, + rsaParameters: func() *tpm2.RSAParams { + tmp := *defaultRsaParams + tmp.ModulusRaw = []byte{0, 1, 2, 3} + return &tmp + }(), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + ak := HCLAkValidator{} + err = ak.Validate(tc.runtimeDataRaw, tc.reportData, tc.rsaParameters) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + } + }) + } +} + +// TestGetHCLAttestationKey is a basic smoke test that only checks if GetAttestationKey can be run error free. +// Testing anything else will only verify that the simulator works as expected, since GetAttestationKey +// only retrieves the attestation key from the TPM. +func TestGetHCLAttestationKey(t *testing.T) { + cgo := os.Getenv("CGO_ENABLED") + if cgo == "0" { + t.Skip("skipping test because CGO is disabled and tpm simulator requires it") + } + require := require.New(t) + assert := assert.New(t) + + tpm, err := simulator.OpenSimulatedTPM() + require.NoError(err) + defer tpm.Close() + + // we should receive an error if no key was saved at index `tpmAkIdx` + _, err = GetAttestationKey(tpm) + assert.Error(err) + + // create a key at the index + tpmAk, err := tpmclient.NewCachedKey(tpm, tpm2.HandleOwner, tpm2.Public{ + Type: tpm2.AlgRSA, + NameAlg: tpm2.AlgSHA256, + Attributes: tpm2.FlagFixedTPM | tpm2.FlagFixedParent | tpm2.FlagSensitiveDataOrigin | tpm2.FlagUserWithAuth | tpm2.FlagNoDA | tpm2.FlagRestricted | tpm2.FlagSign, + RSAParameters: &tpm2.RSAParams{ + Sign: &tpm2.SigScheme{ + Alg: tpm2.AlgRSASSA, + Hash: tpm2.AlgSHA256, + }, + KeyBits: 2048, + }, + }, tpmAkIdx) + require.NoError(err) + defer tpmAk.Close() + + // we should now be able to retrieve the key + _, err = GetAttestationKey(tpm) + assert.NoError(err) +} diff --git a/internal/attestation/azure/snp/BUILD.bazel b/internal/attestation/azure/snp/BUILD.bazel index 7adea0c3e..abb300961 100644 --- a/internal/attestation/azure/snp/BUILD.bazel +++ b/internal/attestation/azure/snp/BUILD.bazel @@ -14,6 +14,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/attestation", + "//internal/attestation/azure", "//internal/attestation/idkeydigest", "//internal/attestation/snp", "//internal/attestation/variant", @@ -28,7 +29,6 @@ go_library( "@com_github_google_go_sev_guest//verify", "@com_github_google_go_sev_guest//verify/trust", "@com_github_google_go_tpm//legacy/tpm2", - "@com_github_google_go_tpm_tools//client", "@com_github_google_go_tpm_tools//proto/attest", ], ) @@ -39,7 +39,6 @@ go_test( "issuer_test.go", "validator_test.go", ], - data = glob(["testdata/**"]), embed = [":snp"], # keep gotags = select({ diff --git a/internal/attestation/azure/snp/imds.go b/internal/attestation/azure/snp/imds.go index 9b1c24cef..8a5547637 100644 --- a/internal/attestation/azure/snp/imds.go +++ b/internal/attestation/azure/snp/imds.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp diff --git a/internal/attestation/azure/snp/issuer.go b/internal/attestation/azure/snp/issuer.go index 3280a731c..b3ab8bdf7 100644 --- a/internal/attestation/azure/snp/issuer.go +++ b/internal/attestation/azure/snp/issuer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -13,15 +13,13 @@ import ( "io" "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/azure" "github.com/edgelesssys/constellation/v2/internal/attestation/snp" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" "github.com/edgelesssys/go-azguestattestation/maa" - tpmclient "github.com/google/go-tpm-tools/client" ) -const tpmAkIdx = 0x81000003 - // Issuer for Azure TPM attestation. type Issuer struct { variant.AzureSEVSNP @@ -40,7 +38,7 @@ func NewIssuer(log attestation.Logger) *Issuer { i.Issuer = vtpm.NewIssuer( vtpm.OpenVTPM, - getAttestationKey, + azure.GetAttestationKey, i.getInstanceInfo, log, ) @@ -83,16 +81,6 @@ func (i *Issuer) getInstanceInfo(ctx context.Context, tpm io.ReadWriteCloser, us return statement, nil } -// getAttestationKey reads the attestation key put into the TPM during early boot. -func getAttestationKey(tpm io.ReadWriter) (*tpmclient.Key, error) { - ak, err := tpmclient.LoadCachedKey(tpm, tpmAkIdx, tpmclient.NullSession{}) - if err != nil { - return nil, fmt.Errorf("reading HCL attestation key from TPM: %w", err) - } - - return ak, nil -} - type imdsAPI interface { getMAAURL(ctx context.Context) (string, error) } diff --git a/internal/attestation/azure/snp/issuer_test.go b/internal/attestation/azure/snp/issuer_test.go index 81f6d6df1..45116303c 100644 --- a/internal/attestation/azure/snp/issuer_test.go +++ b/internal/attestation/azure/snp/issuer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -11,14 +11,10 @@ import ( "encoding/json" "errors" "io" - "os" "testing" - "github.com/edgelesssys/constellation/v2/internal/attestation/simulator" "github.com/edgelesssys/constellation/v2/internal/attestation/snp" "github.com/edgelesssys/go-azguestattestation/maa" - tpmclient "github.com/google/go-tpm-tools/client" - "github.com/google/go-tpm/legacy/tpm2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -86,7 +82,7 @@ func TestGetSNPAttestation(t *testing.T) { data := []byte("data") - attestationJSON, err := issuer.getInstanceInfo(context.Background(), nil, data) + attestationJSON, err := issuer.getInstanceInfo(t.Context(), nil, data) if tc.wantErr { assert.Error(err) return @@ -113,46 +109,6 @@ func TestGetSNPAttestation(t *testing.T) { } } -// TestGetHCLAttestationKey is a basic smoke test that only checks if getAkPub can be run error free. -// Testing anything else will only verify that the simulator works as expected, since getAkPub -// only retrieves the attestation key from the TPM. -func TestGetHCLAttestationKey(t *testing.T) { - cgo := os.Getenv("CGO_ENABLED") - if cgo == "0" { - t.Skip("skipping test because CGO is disabled and tpm simulator requires it") - } - require := require.New(t) - assert := assert.New(t) - - tpm, err := simulator.OpenSimulatedTPM() - require.NoError(err) - defer tpm.Close() - - // we should receive an error if no key was saved at index `tpmAkIdx` - _, err = getAttestationKey(tpm) - assert.Error(err) - - // create a key at the index - tpmAk, err := tpmclient.NewCachedKey(tpm, tpm2.HandleOwner, tpm2.Public{ - Type: tpm2.AlgRSA, - NameAlg: tpm2.AlgSHA256, - Attributes: tpm2.FlagFixedTPM | tpm2.FlagFixedParent | tpm2.FlagSensitiveDataOrigin | tpm2.FlagUserWithAuth | tpm2.FlagNoDA | tpm2.FlagRestricted | tpm2.FlagSign, - RSAParameters: &tpm2.RSAParams{ - Sign: &tpm2.SigScheme{ - Alg: tpm2.AlgRSASSA, - Hash: tpm2.AlgSHA256, - }, - KeyBits: 2048, - }, - }, tpmAkIdx) - require.NoError(err) - defer tpmAk.Close() - - // we should now be able to retrieve the key - _, err = getAttestationKey(tpm) - assert.NoError(err) -} - type stubImdsClient struct { maaURL string apiError error diff --git a/internal/attestation/azure/snp/maa.go b/internal/attestation/azure/snp/maa.go index 2cf82766a..a51348925 100644 --- a/internal/attestation/azure/snp/maa.go +++ b/internal/attestation/azure/snp/maa.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp diff --git a/internal/attestation/azure/snp/snp.go b/internal/attestation/azure/snp/snp.go index 8c109d25a..84895748b 100644 --- a/internal/attestation/azure/snp/snp.go +++ b/internal/attestation/azure/snp/snp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/azure/snp/validator.go b/internal/attestation/azure/snp/validator.go index 856e528ff..701f526ff 100644 --- a/internal/attestation/azure/snp/validator.go +++ b/internal/attestation/azure/snp/validator.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -10,15 +10,13 @@ import ( "bytes" "context" "crypto" - "crypto/sha256" "crypto/x509" - "encoding/base64" - "encoding/binary" "encoding/json" "errors" "fmt" "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/azure" "github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest" "github.com/edgelesssys/constellation/v2/internal/attestation/snp" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" @@ -78,7 +76,7 @@ func NewValidator(cfg *config.AzureSEVSNP, log attestation.Logger) *Validator { log = nopAttestationLogger{} } v := &Validator{ - hclValidator: &attestationKey{}, + hclValidator: &azure.HCLAkValidator{}, maa: newMAAClient(), config: cfg, log: log, @@ -118,25 +116,11 @@ func (v *Validator) getTrustedKey(ctx context.Context, attDoc vtpm.AttestationDo return nil, fmt.Errorf("parsing attestation report: %w", err) } - // ASK, as cached in joinservice or reported from THIM / KDS. - ask, err := x509.ParseCertificate(att.CertificateChain.AskCert) + verifyOpts, err := getVerifyOpts(att) if err != nil { - return nil, fmt.Errorf("parsing ASK certificate: %w", err) + return nil, fmt.Errorf("getting verify options: %w", err) } - verifyOpts := &verify.Options{ - TrustedRoots: map[string][]*trust.AMDRootCerts{ - "Milan": { - { - Product: "Milan", - ProductCerts: &trust.ProductCerts{ - Ask: ask, - Ark: trustedArk, - }, - }, - }, - }, - } if err := v.attestationVerifier.SNPAttestation(att, verifyOpts); err != nil { return nil, fmt.Errorf("verifying SNP attestation: %w", err) } @@ -191,7 +175,7 @@ func (v *Validator) getTrustedKey(ctx context.Context, attDoc vtpm.AttestationDo if err != nil { return nil, err } - if err = v.hclValidator.validate(instanceInfo.Azure.RuntimeData, att.Report.ReportData, pubArea.RSAParameters); err != nil { + if err = v.hclValidator.Validate(instanceInfo.Azure.RuntimeData, att.Report.ReportData, pubArea.RSAParameters); err != nil { return nil, fmt.Errorf("validating HCLAkPub: %w", err) } @@ -214,18 +198,18 @@ func (v *Validator) checkIDKeyDigest(ctx context.Context, report *spb.Attestatio // the MAA if necessary. switch v.config.FirmwareSignerConfig.EnforcementPolicy { case idkeydigest.MAAFallback: - v.log.Infof( + v.log.Info(fmt.Sprintf( "Configured idkeydigests %x don't contain reported idkeydigest %x, falling back to MAA validation", v.config.FirmwareSignerConfig.AcceptedKeyDigests, report.Report.IdKeyDigest, - ) + )) return v.maa.validateToken(ctx, v.config.FirmwareSignerConfig.MAAURL, maaToken, extraData) case idkeydigest.WarnOnly: - v.log.Warnf( + v.log.Warn(fmt.Sprintf( "Configured idkeydigests %x don't contain reported idkeydigest %x", v.config.FirmwareSignerConfig.AcceptedKeyDigests, report.Report.IdKeyDigest, - ) + )) default: return fmt.Errorf( "configured idkeydigests %x don't contain reported idkeydigest %x", @@ -238,79 +222,47 @@ func (v *Validator) checkIDKeyDigest(ctx context.Context, report *spb.Attestatio return nil } -type attestationKey struct { - PublicPart []akPub `json:"keys"` -} - -// validate validates that the attestation key from the TPM is trustworthy. The steps are: -// 1. runtime data read from the TPM has the same sha256 digest as reported in `report_data` of the SNP report. -// 2. modulus reported in runtime data matches modulus from key at idx 0x81000003. -// 3. exponent reported in runtime data matches exponent from key at idx 0x81000003. -// The function is currently tested manually on a Azure Ubuntu CVM. -func (a *attestationKey) validate(runtimeDataRaw []byte, reportData []byte, rsaParameters *tpm2.RSAParams) error { - if err := json.Unmarshal(runtimeDataRaw, a); err != nil { - return fmt.Errorf("unmarshalling json: %w", err) - } - - sum := sha256.Sum256(runtimeDataRaw) - if len(reportData) < len(sum) { - return fmt.Errorf("reportData has unexpected size: %d", len(reportData)) - } - if !bytes.Equal(sum[:], reportData[:len(sum)]) { - return errors.New("unexpected runtimeData digest in TPM") - } - - if len(a.PublicPart) < 1 { - return errors.New("did not receive any keys in runtime data") - } - rawN, err := base64.RawURLEncoding.DecodeString(a.PublicPart[0].N) - if err != nil { - return fmt.Errorf("decoding modulus string: %w", err) - } - if !bytes.Equal(rawN, rsaParameters.ModulusRaw) { - return fmt.Errorf("unexpected modulus value in TPM") - } - - rawE, err := base64.RawURLEncoding.DecodeString(a.PublicPart[0].E) - if err != nil { - return fmt.Errorf("decoding exponent string: %w", err) - } - paddedRawE := make([]byte, 4) - copy(paddedRawE, rawE) - exponent := binary.LittleEndian.Uint32(paddedRawE) - - // According to this comment [1] the TPM uses "0" to represent the default exponent "65537". - // The go tpm library also reports the exponent as 0. Thus we have to handle it specially. - // [1] https://github.com/tpm2-software/tpm2-tools/pull/1973#issue-596685005 - if !((exponent == 65537 && rsaParameters.ExponentRaw == 0) || exponent == rsaParameters.ExponentRaw) { - return fmt.Errorf("unexpected N value in TPM") - } - - return nil -} - -// hclAkValidator validates an attestation key issued by the Host Compatibility Layer (HCL). -// The HCL is written by Azure, and sits between the Hypervisor and CVM OS. -// The HCL runs in the protected context of the CVM. -type hclAkValidator interface { - validate(runtimeDataRaw []byte, reportData []byte, rsaParameters *tpm2.RSAParams) error -} - -// akPub are the public parameters of an RSA attestation key. -type akPub struct { - E string `json:"e"` - N string `json:"n"` -} - // nopAttestationLogger is a no-op implementation of AttestationLogger. type nopAttestationLogger struct{} // Infof is a no-op. -func (nopAttestationLogger) Infof(string, ...interface{}) {} +func (nopAttestationLogger) Info(string, ...interface{}) {} // Warnf is a no-op. -func (nopAttestationLogger) Warnf(string, ...interface{}) {} +func (nopAttestationLogger) Warn(string, ...interface{}) {} type maaValidator interface { validateToken(ctx context.Context, maaURL string, token string, extraData []byte) error } + +type hclAkValidator interface { + Validate(runtimeDataRaw []byte, reportData []byte, rsaParameters *tpm2.RSAParams) error +} + +func getVerifyOpts(att *spb.Attestation) (*verify.Options, error) { + // ASK, as cached in joinservice or reported from THIM / KDS. + ask, err := x509.ParseCertificate(att.CertificateChain.AskCert) + if err != nil { + return nil, fmt.Errorf("parsing ASK certificate: %w", err) + } + ark, err := x509.ParseCertificate(att.CertificateChain.ArkCert) + if err != nil { + return nil, fmt.Errorf("parsing ARK certificate: %w", err) + } + + verifyOpts := &verify.Options{ + TrustedRoots: map[string][]*trust.AMDRootCerts{ + "Milan": { + { + Product: "Milan", + ProductCerts: &trust.ProductCerts{ + Ask: ask, + Ark: ark, + }, + }, + }, + }, + } + + return verifyOpts, nil +} diff --git a/internal/attestation/azure/snp/validator_test.go b/internal/attestation/azure/snp/validator_test.go index 2b2c9a241..9b382fb08 100644 --- a/internal/attestation/azure/snp/validator_test.go +++ b/internal/attestation/azure/snp/validator_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -10,8 +10,6 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/base64" - "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -23,7 +21,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation" "github.com/edgelesssys/constellation/v2/internal/attestation/idkeydigest" "github.com/edgelesssys/constellation/v2/internal/attestation/simulator" - "github.com/edgelesssys/constellation/v2/internal/attestation/snp" "github.com/edgelesssys/constellation/v2/internal/attestation/snp/testdata" "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" "github.com/edgelesssys/constellation/v2/internal/config" @@ -60,7 +57,7 @@ func TestNewValidator(t *testing.T) { } for name, tc := range testCases { - t.Run(name, func(t *testing.T) { + t.Run(name, func(_ *testing.T) { validator := NewValidator(tc.cfg, tc.logger) require.NotNil(validator) require.NotNil(validator.log) @@ -185,7 +182,7 @@ func TestCheckIDKeyDigest(t *testing.T) { report := reportWithIDKeyDigest(tc.idKeyDigest) validator := newTestValidator(cfg, tc.validateMaaTokenErr) - err := validator.checkIDKeyDigest(context.Background(), report, "", nil) + err := validator.checkIDKeyDigest(t.Context(), report, "", nil) if tc.wantErr { require.Error(err) } else { @@ -203,107 +200,6 @@ func (v *stubMaaValidator) validateToken(_ context.Context, _ string, _ string, return v.validateTokenErr } -// TestValidateAk tests the attestation key validation with a simulated TPM device. -func TestValidateAk(t *testing.T) { - cgo := os.Getenv("CGO_ENABLED") - if cgo == "0" { - t.Skip("skipping test because CGO is disabled and tpm simulator requires it") - } - - int32ToBytes := func(val uint32) []byte { - r := make([]byte, 4) - binary.PutUvarint(r, uint64(val)) - return r - } - - require := require.New(t) - - tpm, err := simulator.OpenSimulatedTPM() - require.NoError(err) - defer tpm.Close() - key, err := client.AttestationKeyRSA(tpm) - require.NoError(err) - defer key.Close() - - e := base64.RawURLEncoding.EncodeToString(int32ToBytes(key.PublicArea().RSAParameters.ExponentRaw)) - n := base64.RawURLEncoding.EncodeToString(key.PublicArea().RSAParameters.ModulusRaw) - - ak := akPub{E: e, N: n} - runtimeData := attestationKey{PublicPart: []akPub{ak}} - - defaultRuntimeDataRaw, err := json.Marshal(runtimeData) - require.NoError(err) - defaultInstanceInfo := snp.InstanceInfo{Azure: &snp.AzureInstanceInfo{RuntimeData: defaultRuntimeDataRaw}} - - sig := sha256.Sum256(defaultRuntimeDataRaw) - defaultReportData := sig[:] - defaultRsaParams := key.PublicArea().RSAParameters - - testCases := map[string]struct { - instanceInfo snp.InstanceInfo - runtimeDataRaw []byte - reportData []byte - rsaParameters *tpm2.RSAParams - wantErr bool - }{ - "success": { - instanceInfo: defaultInstanceInfo, - runtimeDataRaw: defaultRuntimeDataRaw, - reportData: defaultReportData, - rsaParameters: defaultRsaParams, - }, - "invalid json": { - instanceInfo: defaultInstanceInfo, - runtimeDataRaw: []byte(""), - reportData: defaultReportData, - rsaParameters: defaultRsaParams, - wantErr: true, - }, - "invalid hash": { - instanceInfo: defaultInstanceInfo, - runtimeDataRaw: defaultRuntimeDataRaw, - reportData: bytes.Repeat([]byte{0}, 64), - rsaParameters: defaultRsaParams, - wantErr: true, - }, - "invalid E": { - instanceInfo: defaultInstanceInfo, - runtimeDataRaw: defaultRuntimeDataRaw, - reportData: defaultReportData, - rsaParameters: func() *tpm2.RSAParams { - tmp := *defaultRsaParams - tmp.ExponentRaw = 1 - return &tmp - }(), - wantErr: true, - }, - "invalid N": { - instanceInfo: defaultInstanceInfo, - runtimeDataRaw: defaultRuntimeDataRaw, - reportData: defaultReportData, - rsaParameters: func() *tpm2.RSAParams { - tmp := *defaultRsaParams - tmp.ModulusRaw = []byte{0, 1, 2, 3} - return &tmp - }(), - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - ak := attestationKey{} - err = ak.validate(tc.runtimeDataRaw, tc.reportData, tc.rsaParameters) - if tc.wantErr { - assert.Error(err) - } else { - assert.NoError(err) - } - }) - } -} - // TestGetTrustedKey tests the verification and validation of attestation report. func TestTrustedKeyFromSNP(t *testing.T) { cgo := os.Getenv("CGO_ENABLED") @@ -472,7 +368,7 @@ func TestTrustedKeyFromSNP(t *testing.T) { ), wantErr: true, assertion: func(assert *assert.Assertions, err error) { - assert.ErrorContains(err, "could not interpret VCEK DER bytes: x509: malformed certificate") + assert.ErrorContains(err, "x509: malformed certificate") }, }, "invalid certchain fall back to embedded": { @@ -754,7 +650,7 @@ func TestTrustedKeyFromSNP(t *testing.T) { attestationValidator: tc.validator, } - key, err := validator.getTrustedKey(context.Background(), attDoc, nil) + key, err := validator.getTrustedKey(t.Context(), attDoc, nil) if tc.wantErr { assert.Error(err) if tc.assertion != nil { @@ -824,11 +720,9 @@ func newStubInstanceInfo(vcek, certChain []byte, report, runtimeData string) (st }, nil } -type stubAttestationKey struct { - PublicPart []akPub -} +type stubAttestationKey struct{} -func (s *stubAttestationKey) validate(runtimeDataRaw []byte, reportData []byte, _ *tpm2.RSAParams) error { +func (s *stubAttestationKey) Validate(runtimeDataRaw []byte, reportData []byte, _ *tpm2.RSAParams) error { if err := json.Unmarshal(runtimeDataRaw, s); err != nil { return fmt.Errorf("unmarshalling json: %w", err) } diff --git a/internal/attestation/azure/tdx/BUILD.bazel b/internal/attestation/azure/tdx/BUILD.bazel new file mode 100644 index 000000000..50feb12aa --- /dev/null +++ b/internal/attestation/azure/tdx/BUILD.bazel @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "tdx", + srcs = [ + "issuer.go", + "tdx.go", + "validator.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/azure/tdx", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/attestation", + "//internal/attestation/azure", + "//internal/attestation/variant", + "//internal/attestation/vtpm", + "//internal/config", + "@com_github_google_go_tdx_guest//abi", + "@com_github_google_go_tdx_guest//proto/tdx", + "@com_github_google_go_tdx_guest//validate", + "@com_github_google_go_tdx_guest//verify", + "@com_github_google_go_tdx_guest//verify/trust", + "@com_github_google_go_tpm//legacy/tpm2", + "@com_github_google_go_tpm_tools//proto/attest", + ], +) + +go_test( + name = "tdx_test", + srcs = ["issuer_test.go"], + embed = [":tdx"], + deps = [ + "//internal/attestation/azure/tdx/testdata", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/internal/attestation/azure/tdx/issuer.go b/internal/attestation/azure/tdx/issuer.go new file mode 100644 index 000000000..1cb051694 --- /dev/null +++ b/internal/attestation/azure/tdx/issuer.go @@ -0,0 +1,179 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package tdx + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/azure" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + "github.com/google/go-tpm/legacy/tpm2" +) + +const ( + imdsURL = "http://169.254.169.254/acc/tdquote" + indexHCLReport = 0x1400001 + hclDataOffset = 1216 + hclReportTypeOffset = 8 + hclReportTypeOffsetStart = hclDataOffset + hclReportTypeOffset + hclRequestDataSizeOffset = 16 + runtimeDataSizeOffset = hclDataOffset + hclRequestDataSizeOffset + hclRequestDataOffset = 20 + runtimeDataOffset = hclDataOffset + hclRequestDataOffset + tdReportSize = 1024 + hwReportStart = 32 + hwReportEnd = 1216 +) + +const ( + hclReportTypeInvalid uint32 = iota + hclReportTypeReserved + hclReportTypeSNP + hclReportTypeTVM + hclReportTypeTDX +) + +// Issuer for Azure confidential VM attestation using TDX. +type Issuer struct { + variant.AzureTDX + *vtpm.Issuer + + quoteGetter quoteGetter +} + +// NewIssuer initializes a new Azure Issuer. +func NewIssuer(log attestation.Logger) *Issuer { + i := &Issuer{ + quoteGetter: imdsQuoteGetter{ + client: &http.Client{Transport: &http.Transport{Proxy: nil}}, + }, + } + + i.Issuer = vtpm.NewIssuer( + vtpm.OpenVTPM, + azure.GetAttestationKey, + i.getInstanceInfo, + log, + ) + return i +} + +func (i *Issuer) getInstanceInfo(ctx context.Context, tpm io.ReadWriteCloser, _ []byte) ([]byte, error) { + // Read HCL report from TPM + report, err := tpm2.NVReadEx(tpm, indexHCLReport, tpm2.HandleOwner, "", 0) + if err != nil { + return nil, err + } + + // Parse the report from the TPM + hwReport, runtimeData, err := parseHCLReport(report) + if err != nil { + return nil, fmt.Errorf("getting HCL report: %w", err) + } + + // Get quote from IMDS API + quote, err := i.quoteGetter.getQuote(ctx, hwReport) + if err != nil { + return nil, fmt.Errorf("getting quote: %w", err) + } + + instanceInfo := InstanceInfo{ + AttestationReport: quote, + RuntimeData: runtimeData, + } + instanceInfoJSON, err := json.Marshal(instanceInfo) + if err != nil { + return nil, fmt.Errorf("marshalling instance info: %w", err) + } + return instanceInfoJSON, nil +} + +func parseHCLReport(report []byte) (hwReport, runtimeData []byte, err error) { + // First, ensure the extracted report is actually for TDX + if len(report) < hclReportTypeOffsetStart+4 { + return nil, nil, fmt.Errorf("invalid HCL report: expected at least %d bytes to read HCL report type, got %d", hclReportTypeOffsetStart+4, len(report)) + } + reportType := binary.LittleEndian.Uint32(report[hclReportTypeOffsetStart : hclReportTypeOffsetStart+4]) + if reportType != hclReportTypeTDX { + return nil, nil, fmt.Errorf("invalid HCL report type: expected TDX (%d), got %d", hclReportTypeTDX, reportType) + } + + // We need the td report (generally called HW report in Azure's samples) from the HCL report to send to the IMDS API + if len(report) < hwReportStart+tdReportSize { + return nil, nil, fmt.Errorf("invalid HCL report: expected at least %d bytes to read td report, got %d", hwReportStart+tdReportSize, len(report)) + } + hwReport = report[hwReportStart : hwReportStart+tdReportSize] + + // We also need the runtime data to verify the attestation key later on the validator side + if len(report) < runtimeDataSizeOffset+4 { + return nil, nil, fmt.Errorf("invalid HCL report: expected at least %d bytes to read runtime data size, got %d", runtimeDataSizeOffset+4, len(report)) + } + runtimeDataSize := int(binary.LittleEndian.Uint32(report[runtimeDataSizeOffset : runtimeDataSizeOffset+4])) + if len(report) < runtimeDataOffset+runtimeDataSize { + return nil, nil, fmt.Errorf("invalid HCL report: expected at least %d bytes to read runtime data, got %d", runtimeDataOffset+runtimeDataSize, len(report)) + } + runtimeData = report[runtimeDataOffset : runtimeDataOffset+runtimeDataSize] + + return hwReport, runtimeData, nil +} + +// imdsQuoteGetter issues TDX quotes using Azure's IMDS API. +type imdsQuoteGetter struct { + client *http.Client +} + +func (i imdsQuoteGetter) getQuote(ctx context.Context, hwReport []byte) ([]byte, error) { + encodedReportJSON, err := json.Marshal(quoteRequest{ + Report: base64.RawURLEncoding.EncodeToString(hwReport), + }) + if err != nil { + return nil, fmt.Errorf("marshalling encoded report: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, imdsURL, bytes.NewReader(encodedReportJSON)) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + req.Header.Add("Content-Type", "application/json") + + res, err := i.client.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + var quoteRes quoteResponse + if err := json.NewDecoder(res.Body).Decode("eRes); err != nil { + return nil, fmt.Errorf("decoding response: %w", err) + } + + return base64.RawURLEncoding.DecodeString(quoteRes.Quote) +} + +type quoteRequest struct { + Report string `json:"report"` +} + +type quoteResponse struct { + Quote string `json:"quote"` +} + +type quoteGetter interface { + getQuote(ctx context.Context, encodedHWReport []byte) ([]byte, error) +} diff --git a/internal/attestation/azure/tdx/issuer_test.go b/internal/attestation/azure/tdx/issuer_test.go new file mode 100644 index 000000000..dcb248aac --- /dev/null +++ b/internal/attestation/azure/tdx/issuer_test.go @@ -0,0 +1,158 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package tdx + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "io" + "net/http" + "testing" + + "github.com/edgelesssys/constellation/v2/internal/attestation/azure/tdx/testdata" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseHCLReport(t *testing.T) { + testCases := map[string]struct { + report []byte + wantErr bool + }{ + "success using testdata": { + report: testdata.HCLReport, + wantErr: false, + }, + "invalid report type": { + report: func() []byte { + report := make([]byte, len(testdata.HCLReport)) + copy(report, testdata.HCLReport) + binary.LittleEndian.PutUint32(report[hclReportTypeOffsetStart:], hclReportTypeInvalid) + return report + }(), + wantErr: true, + }, + "report too short for HCL report type": { + report: func() []byte { + report := make([]byte, hclReportTypeOffsetStart+3) + copy(report, testdata.HCLReport) + return report + }(), + wantErr: true, + }, + "report too short for runtime data size": { + report: func() []byte { + report := make([]byte, runtimeDataSizeOffset+3) + copy(report, testdata.HCLReport) + return report + }(), + wantErr: true, + }, + "runtime data shorter than runtime data size": { + report: func() []byte { + report := make([]byte, len(testdata.HCLReport)) + copy(report, testdata.HCLReport) + // Lets claim the report contains a much larger runtime data entry than it actually does. + // That way, we can easily test if our code correctly handles reports that are shorter than + // what they claim to be and avoid panics. + binary.LittleEndian.PutUint32(report[runtimeDataSizeOffset:], 0xFFFF) + return report + }(), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + hwReport, runtimeData, err := parseHCLReport(tc.report) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + assert.Equal(tc.report[hwReportStart:hwReportStart+tdReportSize], hwReport) + assert.Greater(len(runtimeData), 0) + assert.Equal(tc.report[runtimeDataOffset:runtimeDataOffset+len(runtimeData)], runtimeData) + }) + } +} + +func TestIMDSGetQuote(t *testing.T) { + testCases := map[string]struct { + client *http.Client + wantErr bool + }{ + "success": { + client: newTestClient(func(_ *http.Request) *http.Response { + quote := quoteResponse{ + Quote: "test", + } + b, err := json.Marshal(quote) + require.NoError(t, err) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(b)), + } + }, + ), + wantErr: false, + }, + "bad status code": { + client: newTestClient(func(_ *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: io.NopCloser(bytes.NewBufferString("")), + } + }, + ), + wantErr: true, + }, + "bad json": { + client: newTestClient(func(_ *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString("")), + } + }, + ), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + quoteGetter := imdsQuoteGetter{ + client: tc.client, + } + + _, err := quoteGetter.getQuote(t.Context(), []byte("test")) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + } + }) + } +} + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +// newTestClient returns *http.Client with Transport replaced to avoid making real calls. +func newTestClient(fn roundTripFunc) *http.Client { + return &http.Client{ + Transport: fn, + } +} diff --git a/internal/attestation/azure/tdx/tdx.go b/internal/attestation/azure/tdx/tdx.go new file mode 100644 index 000000000..cbf34b4a4 --- /dev/null +++ b/internal/attestation/azure/tdx/tdx.go @@ -0,0 +1,26 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +/* +package tdx implements attestation for TDX on Azure. + +Quotes are generated using an Azure provided vTPM and the IMDS API. +They are verified using the go-tdx-guest library. + +More specifically: +- The vTPM is used to collected a TPM attestation and a Hardware Compatibility Layer (HCL) report. +- The HCL report is sent to the IMDS API to generate a TDX quote. +- The quote is verified using the go-tdx-guest library. +- The quote's report data can be used to verify the TPM's attestation key. +- The attestation key can be used to verify the TPM attestation. +*/ +package tdx + +// InstanceInfo wraps the TDX report with additional Azure specific runtime data. +type InstanceInfo struct { + AttestationReport []byte + RuntimeData []byte +} diff --git a/internal/attestation/azure/tdx/testdata/BUILD.bazel b/internal/attestation/azure/tdx/testdata/BUILD.bazel new file mode 100644 index 000000000..cf880a0aa --- /dev/null +++ b/internal/attestation/azure/tdx/testdata/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "testdata", + srcs = ["testdata.go"], + embedsrcs = ["hclreport.bin"], + importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/azure/tdx/testdata", + visibility = ["//:__subpackages__"], +) diff --git a/internal/attestation/azure/tdx/testdata/hclreport.bin b/internal/attestation/azure/tdx/testdata/hclreport.bin new file mode 100644 index 000000000..ff9494a0f Binary files /dev/null and b/internal/attestation/azure/tdx/testdata/hclreport.bin differ diff --git a/internal/attestation/azure/tdx/testdata/testdata.go b/internal/attestation/azure/tdx/testdata/testdata.go new file mode 100644 index 000000000..08902b9eb --- /dev/null +++ b/internal/attestation/azure/tdx/testdata/testdata.go @@ -0,0 +1,15 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +// Package testdata contains testing data for an attestation process. +package testdata + +import _ "embed" + +// HCLReport is an example HCL report from an Azure TDX VM. +// +//go:embed hclreport.bin +var HCLReport []byte diff --git a/internal/attestation/azure/tdx/validator.go b/internal/attestation/azure/tdx/validator.go new file mode 100644 index 000000000..2dedf2390 --- /dev/null +++ b/internal/attestation/azure/tdx/validator.go @@ -0,0 +1,124 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package tdx + +import ( + "context" + "crypto" + "crypto/x509" + "encoding/json" + "fmt" + + "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/azure" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/google/go-tdx-guest/abi" + "github.com/google/go-tdx-guest/proto/tdx" + "github.com/google/go-tdx-guest/validate" + "github.com/google/go-tdx-guest/verify" + "github.com/google/go-tdx-guest/verify/trust" + "github.com/google/go-tpm-tools/proto/attest" + "github.com/google/go-tpm/legacy/tpm2" +) + +// Validator for Azure confidential VM attestation using TDX. +type Validator struct { + variant.AzureTDX + *vtpm.Validator + cfg *config.AzureTDX + + getter trust.HTTPSGetter + hclValidator hclAkValidator +} + +// NewValidator returns a new Validator for Azure confidential VM attestation using TDX. +func NewValidator(cfg *config.AzureTDX, log attestation.Logger) *Validator { + v := &Validator{ + cfg: cfg, + getter: trust.DefaultHTTPSGetter(), + hclValidator: &azure.HCLAkValidator{}, + } + + v.Validator = vtpm.NewValidator( + cfg.Measurements, + v.getTrustedTPMKey, + func(vtpm.AttestationDocument, *attest.MachineState) error { + return nil + }, + log, + ) + + return v +} + +func (v *Validator) getTrustedTPMKey(_ context.Context, attDoc vtpm.AttestationDocument, _ []byte) (crypto.PublicKey, error) { + var instanceInfo InstanceInfo + if err := json.Unmarshal(attDoc.InstanceInfo, &instanceInfo); err != nil { + return nil, err + } + + quotePb, err := abi.QuoteToProto(instanceInfo.AttestationReport) + if err != nil { + return nil, err + } + quote, ok := quotePb.(*tdx.QuoteV4) + if !ok { + return nil, fmt.Errorf("unexpected quote type: %T", quote) + } + + if err := v.validateQuote(quote); err != nil { + return nil, err + } + + // Decode the public area of the attestation key and validate its trustworthiness. + pubArea, err := tpm2.DecodePublic(attDoc.Attestation.AkPub) + if err != nil { + return nil, err + } + if err = v.hclValidator.Validate(instanceInfo.RuntimeData, quote.TdQuoteBody.ReportData, pubArea.RSAParameters); err != nil { + return nil, fmt.Errorf("validating HCLAkPub: %w", err) + } + + return pubArea.Key() +} + +func (v *Validator) validateQuote(tdxQuote *tdx.QuoteV4) error { + roots := x509.NewCertPool() + roots.AddCert((*x509.Certificate)(&v.cfg.IntelRootKey)) + + if err := verify.TdxQuote(tdxQuote, &verify.Options{ + CheckRevocations: true, + GetCollateral: true, + TrustedRoots: roots, + Getter: v.getter, + }); err != nil { + return err + } + + if err := validate.TdxQuote(tdxQuote, &validate.Options{ + HeaderOptions: validate.HeaderOptions{ + MinimumQeSvn: v.cfg.QESVN.Value, + MinimumPceSvn: v.cfg.PCESVN.Value, + QeVendorID: v.cfg.QEVendorID.Value, + }, + TdQuoteBodyOptions: validate.TdQuoteBodyOptions{ + MinimumTeeTcbSvn: v.cfg.TEETCBSVN.Value, + MrSeam: v.cfg.MRSeam, + Xfam: v.cfg.XFAM.Value, + }, + }); err != nil { + return err + } + + return nil +} + +type hclAkValidator interface { + Validate(runtimeDataRaw []byte, reportData []byte, rsaParameters *tpm2.RSAParams) error +} diff --git a/internal/attestation/azure/trustedlaunch/issuer.go b/internal/attestation/azure/trustedlaunch/issuer.go index f62a88104..eaccfc06d 100644 --- a/internal/attestation/azure/trustedlaunch/issuer.go +++ b/internal/attestation/azure/trustedlaunch/issuer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package trustedlaunch diff --git a/internal/attestation/azure/trustedlaunch/trustedlaunch.go b/internal/attestation/azure/trustedlaunch/trustedlaunch.go index 8959bc32f..822ee86e9 100644 --- a/internal/attestation/azure/trustedlaunch/trustedlaunch.go +++ b/internal/attestation/azure/trustedlaunch/trustedlaunch.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/azure/trustedlaunch/trustedlaunch_test.go b/internal/attestation/azure/trustedlaunch/trustedlaunch_test.go index 023bb785f..1fe9a3989 100644 --- a/internal/attestation/azure/trustedlaunch/trustedlaunch_test.go +++ b/internal/attestation/azure/trustedlaunch/trustedlaunch_test.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package trustedlaunch import ( "bytes" - "context" "crypto/rand" "crypto/rsa" "crypto/x509" @@ -104,7 +103,7 @@ func TestGetAttestationCert(t *testing.T) { wantValidateErr bool }{ "success": { - crlServer: func(req *http.Request) *http.Response { + crlServer: func(_ *http.Request) *http.Response { return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(intermediateCert.Raw)), @@ -136,14 +135,14 @@ func TestGetAttestationCert(t *testing.T) { }, }, "intermediate cert cannot be fetched": { - crlServer: func(req *http.Request) *http.Response { + crlServer: func(_ *http.Request) *http.Response { return &http.Response{StatusCode: http.StatusNotFound} }, getAkCert: defaultAkCertFunc, wantIssueErr: true, }, "intermediate cert is not signed by root cert": { - crlServer: func(req *http.Request) *http.Response { + crlServer: func(_ *http.Request) *http.Response { return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(rootCert.Raw)), @@ -153,7 +152,7 @@ func TestGetAttestationCert(t *testing.T) { wantValidateErr: true, }, "ak does not match ak cert public key": { - crlServer: func(req *http.Request) *http.Response { + crlServer: func(_ *http.Request) *http.Response { return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(intermediateCert.Raw)), @@ -192,7 +191,7 @@ func TestGetAttestationCert(t *testing.T) { issuer := NewIssuer(logger.NewTest(t)) issuer.hClient = newTestClient(tc.crlServer) - certs, err := issuer.getAttestationCert(context.Background(), tpm, nil) + certs, err := issuer.getAttestationCert(t.Context(), tpm, nil) if tc.wantIssueErr { assert.Error(err) return @@ -213,7 +212,7 @@ func TestGetAttestationCert(t *testing.T) { roots.AddCert(cert) validator.roots = roots - key, err := validator.verifyAttestationKey(context.Background(), attDoc, nil) + key, err := validator.verifyAttestationKey(t.Context(), attDoc, nil) if tc.wantValidateErr { assert.Error(err) return diff --git a/internal/attestation/azure/trustedlaunch/validator.go b/internal/attestation/azure/trustedlaunch/validator.go index 368db8368..4cdefb580 100644 --- a/internal/attestation/azure/trustedlaunch/validator.go +++ b/internal/attestation/azure/trustedlaunch/validator.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package trustedlaunch diff --git a/internal/attestation/choose/BUILD.bazel b/internal/attestation/choose/BUILD.bazel index f57356c50..09bd9d2b9 100644 --- a/internal/attestation/choose/BUILD.bazel +++ b/internal/attestation/choose/BUILD.bazel @@ -12,8 +12,10 @@ go_library( "//internal/attestation/aws/nitrotpm", "//internal/attestation/aws/snp", "//internal/attestation/azure/snp", + "//internal/attestation/azure/tdx", "//internal/attestation/azure/trustedlaunch", - "//internal/attestation/gcp", + "//internal/attestation/gcp/es", + "//internal/attestation/gcp/snp", "//internal/attestation/qemu", "//internal/attestation/tdx", "//internal/attestation/variant", diff --git a/internal/attestation/choose/choose.go b/internal/attestation/choose/choose.go index a74551860..6918210b2 100644 --- a/internal/attestation/choose/choose.go +++ b/internal/attestation/choose/choose.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package choose @@ -14,8 +14,10 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/aws/nitrotpm" awssnp "github.com/edgelesssys/constellation/v2/internal/attestation/aws/snp" azuresnp "github.com/edgelesssys/constellation/v2/internal/attestation/azure/snp" + azuretdx "github.com/edgelesssys/constellation/v2/internal/attestation/azure/tdx" "github.com/edgelesssys/constellation/v2/internal/attestation/azure/trustedlaunch" - "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp/es" + gcpsnp "github.com/edgelesssys/constellation/v2/internal/attestation/gcp/snp" "github.com/edgelesssys/constellation/v2/internal/attestation/qemu" "github.com/edgelesssys/constellation/v2/internal/attestation/tdx" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" @@ -33,8 +35,12 @@ func Issuer(attestationVariant variant.Variant, log attestation.Logger) (atls.Is return trustedlaunch.NewIssuer(log), nil case variant.AzureSEVSNP{}: return azuresnp.NewIssuer(log), nil + case variant.AzureTDX{}: + return azuretdx.NewIssuer(log), nil case variant.GCPSEVES{}: - return gcp.NewIssuer(log), nil + return es.NewIssuer(log), nil + case variant.GCPSEVSNP{}: + return gcpsnp.NewIssuer(log), nil case variant.QEMUVTPM{}: return qemu.NewIssuer(log), nil case variant.QEMUTDX{}: @@ -57,8 +63,12 @@ func Validator(cfg config.AttestationCfg, log attestation.Logger) (atls.Validato return trustedlaunch.NewValidator(cfg, log), nil case *config.AzureSEVSNP: return azuresnp.NewValidator(cfg, log), nil + case *config.AzureTDX: + return azuretdx.NewValidator(cfg, log), nil case *config.GCPSEVES: - return gcp.NewValidator(cfg, log), nil + return es.NewValidator(cfg, log) + case *config.GCPSEVSNP: + return gcpsnp.NewValidator(cfg, log) case *config.QEMUVTPM: return qemu.NewValidator(cfg, log), nil case *config.QEMUTDX: diff --git a/internal/attestation/choose/choose_test.go b/internal/attestation/choose/choose_test.go index 95d4184fc..6cc20cdb8 100644 --- a/internal/attestation/choose/choose_test.go +++ b/internal/attestation/choose/choose_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package choose @@ -31,12 +31,18 @@ func TestIssuer(t *testing.T) { "azure-sev-snp": { variant: variant.AzureSEVSNP{}, }, + "azure-tdx": { + variant: variant.AzureTDX{}, + }, "azure-trusted-launch": { variant: variant.AzureTrustedLaunch{}, }, "gcp-sev-es": { variant: variant.GCPSEVES{}, }, + "gcp-sev-snp": { + variant: variant.GCPSEVSNP{}, + }, "qemu-vtpm": { variant: variant.QEMUVTPM{}, }, @@ -77,12 +83,18 @@ func TestValidator(t *testing.T) { "azure-sev-snp": { cfg: &config.AzureSEVSNP{}, }, + "azure-tdx": { + cfg: &config.AzureTDX{}, + }, "azure-trusted-launch": { cfg: &config.AzureTrustedLaunch{}, }, "gcp-sev-es": { cfg: &config.GCPSEVES{}, }, + "gcp-sev-snp": { + cfg: &config.GCPSEVSNP{}, + }, "qemu-vtpm": { cfg: &config.QEMUVTPM{}, }, diff --git a/internal/attestation/gcp/BUILD.bazel b/internal/attestation/gcp/BUILD.bazel index 7cabb294b..8b8c24d8c 100644 --- a/internal/attestation/gcp/BUILD.bazel +++ b/internal/attestation/gcp/BUILD.bazel @@ -1,21 +1,18 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") -load("//bazel/go:go_test.bzl", "go_test") go_library( name = "gcp", srcs = [ "gcp.go", - "issuer.go", - "validator.go", + "metadata.go", + "restclient.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/gcp", visibility = ["//:__subpackages__"], deps = [ - "//internal/attestation", + "//internal/attestation/snp", "//internal/attestation/variant", "//internal/attestation/vtpm", - "//internal/config", - "@com_github_google_go_tpm_tools//client", "@com_github_google_go_tpm_tools//proto/attest", "@com_github_googleapis_gax_go_v2//:gax-go", "@com_google_cloud_go_compute//apiv1", @@ -24,22 +21,3 @@ go_library( "@org_golang_google_api//option", ], ) - -go_test( - name = "gcp_test", - srcs = [ - "issuer_test.go", - "validator_test.go", - ], - embed = [":gcp"], - deps = [ - "//internal/attestation/vtpm", - "@com_github_google_go_tpm_tools//proto/attest", - "@com_github_googleapis_gax_go_v2//:gax-go", - "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@com_google_cloud_go_compute//apiv1/computepb", - "@org_golang_google_api//option", - "@org_golang_google_protobuf//proto", - ], -) diff --git a/internal/attestation/gcp/es/BUILD.bazel b/internal/attestation/gcp/es/BUILD.bazel new file mode 100644 index 000000000..a7d089412 --- /dev/null +++ b/internal/attestation/gcp/es/BUILD.bazel @@ -0,0 +1,43 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "es", + srcs = [ + "es.go", + "issuer.go", + "validator.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/gcp/es", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/attestation", + "//internal/attestation/gcp", + "//internal/attestation/variant", + "//internal/attestation/vtpm", + "//internal/config", + "@com_github_google_go_tpm_tools//client", + "@com_github_google_go_tpm_tools//proto/attest", + ], +) + +go_test( + name = "es_test", + srcs = [ + "issuer_test.go", + "validator_test.go", + ], + embed = [":es"], + deps = [ + "//internal/attestation/gcp", + "//internal/attestation/variant", + "//internal/attestation/vtpm", + "@com_github_google_go_tpm_tools//proto/attest", + "@com_github_googleapis_gax_go_v2//:gax-go", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@com_google_cloud_go_compute//apiv1/computepb", + "@org_golang_google_api//option", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/internal/attestation/gcp/es/es.go b/internal/attestation/gcp/es/es.go new file mode 100644 index 000000000..b3c9f2645 --- /dev/null +++ b/internal/attestation/gcp/es/es.go @@ -0,0 +1,45 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +/* +# GCP SEV-ES attestation + +Google offers [confidential VMs], utilizing AMD SEV-ES to provide memory encryption. + +AMD SEV-ES doesn't offer much in terms of remote attestation, and following that the VMs don't offer much either, see [their docs] on how to validate a confidential VM for some insights. +However, each VM comes with a [virtual Trusted Platform Module (vTPM)]. +This module can be used to generate VM unique encryption keys or to attest the platform's chain of boot. We can use the vTPM to verify the VM is running on AMD SEV-ES enabled hardware, allowing us to bootstrap a constellation cluster. + +# Issuer + +Generates a TPM attestation key using a Google provided attestation key. +Additionally project ID, zone, and instance name are fetched from the metadata server and attached to the attestation document. + +# Validator + +Verifies the TPM attestation by using a public key provided by Google's API corresponding to the project ID, zone, instance name tuple attached to the attestation document. + +# Problems + + - SEV-ES is somewhat limited when compared to the newer version SEV-SNP + + Comparison of SEV, SEV-ES, and SEV-SNP can be seen on page seven of [AMD's SNP whitepaper] + + - We have to trust Google + + Since the vTPM is provided by Google, and they could do whatever they want with it, we have no save proof of the VMs actually being confidential. + + - The provided vTPM has no endorsement certificate for its attestation key + + Without a certificate signing the authenticity of any endorsement keys we have no way of establishing a chain of trust. + Instead, we have to rely on Google's API to provide us with the public key of the vTPM's endorsement key. + +[confidential VMs]: https://cloud.google.com/compute/confidential-vm/docs/about-cvm +[their docs]: https://cloud.google.com/compute/confidential-vm/docs/monitoring +[virtual Trusted Platform Module (vTPM)]: https://cloud.google.com/security/shielded-cloud/shielded-vm#vtpm +[AMD's SNP whitepaper]: https://www.amd.com/system/files/TechDocs/SEV-SNP-strengthening-vm-isolation-with-integrity-protection-and-more.pdf#page=7 +*/ +package es diff --git a/internal/attestation/gcp/es/issuer.go b/internal/attestation/gcp/es/issuer.go new file mode 100644 index 000000000..935bc99eb --- /dev/null +++ b/internal/attestation/gcp/es/issuer.go @@ -0,0 +1,33 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package es + +import ( + "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + tpmclient "github.com/google/go-tpm-tools/client" +) + +// Issuer for GCP confidential VM attestation. +type Issuer struct { + variant.GCPSEVES + *vtpm.Issuer +} + +// NewIssuer initializes a new GCP Issuer. +func NewIssuer(log attestation.Logger) *Issuer { + return &Issuer{ + Issuer: vtpm.NewIssuer( + vtpm.OpenVTPM, + tpmclient.GceAttestationKeyRSA, + gcp.GCEInstanceInfo(gcp.MetadataClient{}), + log, + ), + } +} diff --git a/internal/attestation/gcp/issuer_test.go b/internal/attestation/gcp/es/issuer_test.go similarity index 83% rename from internal/attestation/gcp/issuer_test.go rename to internal/attestation/gcp/es/issuer_test.go index 4ad64c7a2..203466f65 100644 --- a/internal/attestation/gcp/issuer_test.go +++ b/internal/attestation/gcp/es/issuer_test.go @@ -1,10 +1,10 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ -package gcp +package es import ( "context" @@ -13,6 +13,7 @@ import ( "io" "testing" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" "github.com/google/go-tpm-tools/proto/attest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -66,7 +67,7 @@ func TestGetGCEInstanceInfo(t *testing.T) { require := require.New(t) var tpm io.ReadWriteCloser - out, err := getGCEInstanceInfo(tc.client)(context.Background(), tpm, nil) + out, err := gcp.GCEInstanceInfo(tc.client)(t.Context(), tpm, nil) if tc.wantErr { assert.Error(err) } else { @@ -90,14 +91,14 @@ type fakeMetadataClient struct { zoneErr error } -func (c fakeMetadataClient) projectID() (string, error) { +func (c fakeMetadataClient) ProjectID(_ context.Context) (string, error) { return c.projectIDString, c.projecIDErr } -func (c fakeMetadataClient) instanceName() (string, error) { +func (c fakeMetadataClient) InstanceName(_ context.Context) (string, error) { return c.instanceNameString, c.instanceNameErr } -func (c fakeMetadataClient) zone() (string, error) { +func (c fakeMetadataClient) Zone(_ context.Context) (string, error) { return c.zoneString, c.zoneErr } diff --git a/internal/attestation/gcp/es/validator.go b/internal/attestation/gcp/es/validator.go new file mode 100644 index 000000000..c846dfdf4 --- /dev/null +++ b/internal/attestation/gcp/es/validator.go @@ -0,0 +1,59 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package es + +import ( + "fmt" + + "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/google/go-tpm-tools/proto/attest" +) + +const minimumGceVersion = 1 + +// Validator for GCP confidential VM attestation. +type Validator struct { + variant.GCPSEVES + *vtpm.Validator +} + +// NewValidator initializes a new GCP validator with the provided PCR values specified in the config. +func NewValidator(cfg *config.GCPSEVES, log attestation.Logger) (*Validator, error) { + getTrustedKey, err := gcp.TrustedKeyGetter(variant.GCPSEVES{}, gcp.NewRESTClient) + if err != nil { + return nil, fmt.Errorf("create trusted key getter: %v", err) + } + + return &Validator{ + Validator: vtpm.NewValidator( + cfg.Measurements, + getTrustedKey, + validateCVM, + log, + ), + }, nil +} + +// validateCVM checks that the machine state represents a GCE AMD-SEV VM. +func validateCVM(_ vtpm.AttestationDocument, state *attest.MachineState) error { + gceVersion := state.Platform.GetGceVersion() + if gceVersion < minimumGceVersion { + return fmt.Errorf("outdated GCE version: %v (require >= %v)", gceVersion, minimumGceVersion) + } + + tech := state.Platform.Technology + wantTech := attest.GCEConfidentialTechnology_AMD_SEV + if tech != wantTech { + return fmt.Errorf("unexpected confidential technology: %v (expected: %v)", tech, wantTech) + } + + return nil +} diff --git a/internal/attestation/gcp/validator_test.go b/internal/attestation/gcp/es/validator_test.go similarity index 91% rename from internal/attestation/gcp/validator_test.go rename to internal/attestation/gcp/es/validator_test.go index 09b9ade4a..fc3783594 100644 --- a/internal/attestation/gcp/validator_test.go +++ b/internal/attestation/gcp/es/validator_test.go @@ -1,10 +1,10 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ -package gcp +package es import ( "context" @@ -14,6 +14,8 @@ import ( "testing" "cloud.google.com/go/compute/apiv1/computepb" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" "github.com/google/go-tpm-tools/proto/attest" "github.com/googleapis/gax-go/v2" @@ -87,7 +89,7 @@ Y+t5OxL3kL15VzY1Ob0d5cMCAwEAAQ== testCases := map[string]struct { instanceInfo []byte - getClient func(ctx context.Context, opts ...option.ClientOption) (gcpRestClient, error) + getClient func(ctx context.Context, opts ...option.ClientOption) (gcp.CVMRestClient, error) wantErr bool }{ "success": { @@ -146,12 +148,12 @@ Y+t5OxL3kL15VzY1Ob0d5cMCAwEAAQ== t.Run(name, func(t *testing.T) { assert := assert.New(t) - v := &Validator{ - restClient: tc.getClient, - } attDoc := vtpm.AttestationDocument{InstanceInfo: tc.instanceInfo} - out, err := v.trustedKeyFromGCEAPI(context.Background(), attDoc, nil) + getTrustedKey, err := gcp.TrustedKeyGetter(variant.GCPSEVES{}, tc.getClient) + require.NoError(t, err) + + out, err := getTrustedKey(t.Context(), attDoc, nil) if tc.wantErr { assert.Error(err) @@ -175,8 +177,8 @@ type fakeInstanceClient struct { ident *computepb.ShieldedInstanceIdentity } -func prepareFakeClient(ident *computepb.ShieldedInstanceIdentity, newErr, getIdentErr error) func(ctx context.Context, opts ...option.ClientOption) (gcpRestClient, error) { - return func(ctx context.Context, opts ...option.ClientOption) (gcpRestClient, error) { +func prepareFakeClient(ident *computepb.ShieldedInstanceIdentity, newErr, getIdentErr error) func(ctx context.Context, opts ...option.ClientOption) (gcp.CVMRestClient, error) { + return func(_ context.Context, _ ...option.ClientOption) (gcp.CVMRestClient, error) { return &fakeInstanceClient{ getIdentErr: getIdentErr, ident: ident, diff --git a/internal/attestation/gcp/gcp.go b/internal/attestation/gcp/gcp.go index 893b002a6..ce84a7222 100644 --- a/internal/attestation/gcp/gcp.go +++ b/internal/attestation/gcp/gcp.go @@ -1,45 +1,10 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* # Google Cloud Platform attestation - -Google offers [confidential VMs], utilizing AMD SEV-ES to provide memory encryption. - -AMD SEV-ES doesn't offer much in terms of remote attestation, and following that the VMs don't offer much either, see [their docs] on how to validate a confidential VM for some insights. -However, each VM comes with a [virtual Trusted Platform Module (vTPM)]. -This module can be used to generate VM unique encryption keys or to attest the platform's chain of boot. We can use the vTPM to verify the VM is running on AMD SEV-ES enabled hardware, allowing us to bootstrap a constellation cluster. - -# Issuer - -Generates a TPM attestation key using a Google provided attestation key. -Additionally project ID, zone, and instance name are fetched from the metadata server and attached to the attestation document. - -# Validator - -Verifies the TPM attestation by using a public key provided by Google's API corresponding to the project ID, zone, instance name tuple attached to the attestation document. - -# Problems - - - SEV-ES is somewhat limited when compared to the newer version SEV-SNP - - Comparison of SEV, SEV-ES, and SEV-SNP can be seen on page seven of [AMD's SNP whitepaper] - - - We have to trust Google - - Since the vTPM is provided by Google, and they could do whatever they want with it, we have no save proof of the VMs actually being confidential. - - - The provided vTPM has no endorsement certificate for its attestation key - - Without a certificate signing the authenticity of any endorsement keys we have no way of establishing a chain of trust. - Instead, we have to rely on Google's API to provide us with the public key of the vTPM's endorsement key. - -[confidential VMs]: https://cloud.google.com/compute/confidential-vm/docs/about-cvm -[their docs]: https://cloud.google.com/compute/confidential-vm/docs/monitoring -[virtual Trusted Platform Module (vTPM)]: https://cloud.google.com/security/shielded-cloud/shielded-vm#vtpm -[AMD's SNP whitepaper]: https://www.amd.com/system/files/TechDocs/SEV-SNP-strengthening-vm-isolation-with-integrity-protection-and-more.pdf#page=7 */ package gcp diff --git a/internal/attestation/gcp/issuer.go b/internal/attestation/gcp/issuer.go deleted file mode 100644 index 4dc36ba0d..000000000 --- a/internal/attestation/gcp/issuer.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "encoding/json" - "errors" - "io" - - "cloud.google.com/go/compute/metadata" - "github.com/edgelesssys/constellation/v2/internal/attestation" - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" - tpmclient "github.com/google/go-tpm-tools/client" - "github.com/google/go-tpm-tools/proto/attest" -) - -// Issuer for GCP confidential VM attestation. -type Issuer struct { - variant.GCPSEVES - *vtpm.Issuer -} - -// NewIssuer initializes a new GCP Issuer. -func NewIssuer(log attestation.Logger) *Issuer { - return &Issuer{ - Issuer: vtpm.NewIssuer( - vtpm.OpenVTPM, - tpmclient.GceAttestationKeyRSA, - getGCEInstanceInfo(metadataClient{}), - log, - ), - } -} - -// getGCEInstanceInfo fetches VM metadata used for attestation. -func getGCEInstanceInfo(client gcpMetadataClient) func(context.Context, io.ReadWriteCloser, []byte) ([]byte, error) { - // Ideally we would want to use the endorsement public key certificate - // However, this is not available on GCE instances - // Workaround: Provide ShieldedVM instance info - // The attestating party can request the VMs signing key using Google's API - return func(context.Context, io.ReadWriteCloser, []byte) ([]byte, error) { - projectID, err := client.projectID() - if err != nil { - return nil, errors.New("unable to fetch projectID") - } - zone, err := client.zone() - if err != nil { - return nil, errors.New("unable to fetch zone") - } - instanceName, err := client.instanceName() - if err != nil { - return nil, errors.New("unable to fetch instance name") - } - - return json.Marshal(&attest.GCEInstanceInfo{ - Zone: zone, - ProjectId: projectID, - InstanceName: instanceName, - }) - } -} - -type gcpMetadataClient interface { - projectID() (string, error) - instanceName() (string, error) - zone() (string, error) -} - -type metadataClient struct{} - -func (c metadataClient) projectID() (string, error) { - return metadata.ProjectID() -} - -func (c metadataClient) instanceName() (string, error) { - return metadata.InstanceName() -} - -func (c metadataClient) zone() (string, error) { - return metadata.Zone() -} diff --git a/internal/attestation/gcp/metadata.go b/internal/attestation/gcp/metadata.go new file mode 100644 index 000000000..f158933db --- /dev/null +++ b/internal/attestation/gcp/metadata.go @@ -0,0 +1,69 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package gcp + +import ( + "context" + "encoding/json" + "errors" + "io" + + "cloud.google.com/go/compute/metadata" + "github.com/google/go-tpm-tools/proto/attest" +) + +// GCEInstanceInfo fetches VM metadata used for attestation from the GCE Metadata API. +func GCEInstanceInfo(client gcpMetadataClient) func(context.Context, io.ReadWriteCloser, []byte) ([]byte, error) { + // Ideally we would want to use the endorsement public key certificate + // However, this is not available on GCE instances + // Workaround: Provide ShieldedVM instance info + // The attesting party can request the VMs signing key using Google's API + return func(ctx context.Context, _ io.ReadWriteCloser, _ []byte) ([]byte, error) { + projectID, err := client.ProjectID(ctx) + if err != nil { + return nil, errors.New("unable to fetch projectID") + } + zone, err := client.Zone(ctx) + if err != nil { + return nil, errors.New("unable to fetch zone") + } + instanceName, err := client.InstanceName(ctx) + if err != nil { + return nil, errors.New("unable to fetch instance name") + } + + return json.Marshal(&attest.GCEInstanceInfo{ + Zone: zone, + ProjectId: projectID, + InstanceName: instanceName, + }) + } +} + +type gcpMetadataClient interface { + ProjectID(context.Context) (string, error) + InstanceName(context.Context) (string, error) + Zone(context.Context) (string, error) +} + +// A MetadataClient fetches metadata from the GCE Metadata API. +type MetadataClient struct{} + +// ProjectID returns the project ID of the GCE instance. +func (c MetadataClient) ProjectID(ctx context.Context) (string, error) { + return metadata.ProjectIDWithContext(ctx) +} + +// InstanceName returns the instance name of the GCE instance. +func (c MetadataClient) InstanceName(ctx context.Context) (string, error) { + return metadata.InstanceNameWithContext(ctx) +} + +// Zone returns the zone the GCE instance is located in. +func (c MetadataClient) Zone(ctx context.Context) (string, error) { + return metadata.ZoneWithContext(ctx) +} diff --git a/internal/attestation/gcp/restclient.go b/internal/attestation/gcp/restclient.go new file mode 100644 index 000000000..dd75b8569 --- /dev/null +++ b/internal/attestation/gcp/restclient.go @@ -0,0 +1,101 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package gcp + +import ( + "context" + "crypto" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + + compute "cloud.google.com/go/compute/apiv1" + "cloud.google.com/go/compute/apiv1/computepb" + "github.com/edgelesssys/constellation/v2/internal/attestation/snp" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + "github.com/google/go-tpm-tools/proto/attest" + "github.com/googleapis/gax-go/v2" + "google.golang.org/api/option" +) + +// RESTClient is a client for the GCE API. +type RESTClient struct { + *compute.InstancesClient +} + +// NewRESTClient creates a new RESTClient. +func NewRESTClient(ctx context.Context, opts ...option.ClientOption) (CVMRestClient, error) { + c, err := compute.NewInstancesRESTClient(ctx, opts...) + if err != nil { + return nil, err + } + return &RESTClient{c}, nil +} + +// CVMRestClient is the interface a GCP REST client for a CVM must implement. +type CVMRestClient interface { + GetShieldedInstanceIdentity(ctx context.Context, req *computepb.GetShieldedInstanceIdentityInstanceRequest, opts ...gax.CallOption) (*computepb.ShieldedInstanceIdentity, error) + Close() error +} + +// TrustedKeyGetter returns a function that queries the GCE API for a shieldedVM's public signing key. +// This key can be used to verify attestation statements issued by the VM. +func TrustedKeyGetter( + attestationVariant variant.Variant, + newRESTClient func(ctx context.Context, opts ...option.ClientOption) (CVMRestClient, error), +) (func(ctx context.Context, attDoc vtpm.AttestationDocument, _ []byte) (crypto.PublicKey, error), error) { + return func(ctx context.Context, attDoc vtpm.AttestationDocument, _ []byte) (crypto.PublicKey, error) { + client, err := newRESTClient(ctx) + if err != nil { + return nil, fmt.Errorf("creating GCE client: %w", err) + } + defer client.Close() + + var gceInstanceInfo attest.GCEInstanceInfo + switch attestationVariant { + case variant.GCPSEVES{}: + if err := json.Unmarshal(attDoc.InstanceInfo, &gceInstanceInfo); err != nil { + return nil, err + } + case variant.GCPSEVSNP{}: + var instanceInfo snp.InstanceInfo + if err := json.Unmarshal(attDoc.InstanceInfo, &instanceInfo); err != nil { + return nil, err + } + gceInstanceInfo = attest.GCEInstanceInfo{ + InstanceName: instanceInfo.GCP.InstanceName, + ProjectId: instanceInfo.GCP.ProjectId, + Zone: instanceInfo.GCP.Zone, + } + default: + return nil, fmt.Errorf("unsupported attestation variant: %v", attestationVariant) + } + + instance, err := client.GetShieldedInstanceIdentity(ctx, &computepb.GetShieldedInstanceIdentityInstanceRequest{ + Instance: gceInstanceInfo.GetInstanceName(), + Project: gceInstanceInfo.GetProjectId(), + Zone: gceInstanceInfo.GetZone(), + }) + if err != nil { + return nil, fmt.Errorf("retrieving VM identity: %w", err) + } + + if instance.SigningKey == nil || instance.SigningKey.EkPub == nil { + return nil, fmt.Errorf("received no signing key from GCP API") + } + + // Parse the signing key return by GetShieldedInstanceIdentity + block, _ := pem.Decode([]byte(*instance.SigningKey.EkPub)) + if block == nil || block.Type != "PUBLIC KEY" { + return nil, fmt.Errorf("failed to decode PEM block containing public key") + } + + return x509.ParsePKIXPublicKey(block.Bytes) + }, nil +} diff --git a/internal/attestation/gcp/snp/BUILD.bazel b/internal/attestation/gcp/snp/BUILD.bazel new file mode 100644 index 000000000..800a69c64 --- /dev/null +++ b/internal/attestation/gcp/snp/BUILD.bazel @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "snp", + srcs = [ + "issuer.go", + "snp.go", + "validator.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/gcp/snp", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/attestation", + "//internal/attestation/gcp", + "//internal/attestation/snp", + "//internal/attestation/variant", + "//internal/attestation/vtpm", + "//internal/config", + "@com_github_google_go_sev_guest//abi", + "@com_github_google_go_sev_guest//kds", + "@com_github_google_go_sev_guest//proto/sevsnp", + "@com_github_google_go_sev_guest//validate", + "@com_github_google_go_sev_guest//verify", + "@com_github_google_go_sev_guest//verify/trust", + "@com_github_google_go_tpm_tools//client", + "@com_github_google_go_tpm_tools//proto/attest", + ], +) diff --git a/internal/attestation/gcp/snp/issuer.go b/internal/attestation/gcp/snp/issuer.go new file mode 100644 index 000000000..a5b26a569 --- /dev/null +++ b/internal/attestation/gcp/snp/issuer.go @@ -0,0 +1,161 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package snp + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "io" + + "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" + "github.com/edgelesssys/constellation/v2/internal/attestation/snp" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + + "github.com/google/go-sev-guest/abi" + "github.com/google/go-tpm-tools/client" + tpmclient "github.com/google/go-tpm-tools/client" + "github.com/google/go-tpm-tools/proto/attest" +) + +// Issuer issues SEV-SNP attestations. +type Issuer struct { + variant.GCPSEVSNP + *vtpm.Issuer +} + +// NewIssuer creates a SEV-SNP based issuer for GCP. +func NewIssuer(log attestation.Logger) *Issuer { + return &Issuer{ + Issuer: vtpm.NewIssuer( + vtpm.OpenVTPM, + getAttestationKey, + getInstanceInfo, + log, + ), + } +} + +// getAttestationKey returns a new attestation key. +func getAttestationKey(tpm io.ReadWriter) (*tpmclient.Key, error) { + tpmAk, err := client.GceAttestationKeyRSA(tpm) + if err != nil { + return nil, fmt.Errorf("creating RSA Endorsement key: %w", err) + } + + return tpmAk, nil +} + +// getInstanceInfo generates an extended SNP report, i.e. the report and any loaded certificates. +// Report generation is triggered by sending ioctl syscalls to the SNP guest device, the AMD PSP generates the report. +// The returned bytes will be written into the attestation document. +func getInstanceInfo(ctx context.Context, _ io.ReadWriteCloser, extraData []byte) ([]byte, error) { + if len(extraData) > 64 { + return nil, fmt.Errorf("extra data too long: %d, should be 64 bytes at most", len(extraData)) + } + var extraData64 [64]byte + copy(extraData64[:], extraData) + + report, certs, err := snp.GetExtendedReport(extraData64) + if err != nil { + return nil, fmt.Errorf("getting extended report: %w", err) + } + + vcek, certChain, err := parseSNPCertTable(certs) + if err != nil { + return nil, fmt.Errorf("parsing vcek: %w", err) + } + + gceInstanceInfo, err := gceInstanceInfo(ctx) + if err != nil { + return nil, fmt.Errorf("getting GCE instance info: %w", err) + } + + raw, err := json.Marshal(snp.InstanceInfo{ + AttestationReport: report, + ReportSigner: vcek, + CertChain: certChain, + GCP: gceInstanceInfo, + }) + if err != nil { + return nil, fmt.Errorf("marshalling instance info: %w", err) + } + + return raw, nil +} + +// gceInstanceInfo returns the instance info for a GCE instance from the metadata API. +func gceInstanceInfo(ctx context.Context) (*attest.GCEInstanceInfo, error) { + c := gcp.MetadataClient{} + + instanceName, err := c.InstanceName(ctx) + if err != nil { + return nil, fmt.Errorf("getting instance name: %w", err) + } + + projectID, err := c.ProjectID(ctx) + if err != nil { + return nil, fmt.Errorf("getting project ID: %w", err) + } + + zone, err := c.Zone(ctx) + if err != nil { + return nil, fmt.Errorf("getting zone: %w", err) + } + + return &attest.GCEInstanceInfo{ + InstanceName: instanceName, + ProjectId: projectID, + Zone: zone, + }, nil +} + +// parseSNPCertTable takes a marshalled SNP certificate table and returns the PEM-encoded VCEK certificate and, +// if present, the ASK of the SNP certificate chain. +// AMD documentation on certificate tables can be found in section 4.1.8.1, revision 2.03 "SEV-ES Guest-Hypervisor Communication Block Standardization". +// https://www.amd.com/content/dam/amd/en/documents/epyc-technical-docs/specifications/56421.pdf +func parseSNPCertTable(certs []byte) (vcekPEM []byte, certChain []byte, err error) { + certTable := abi.CertTable{} + if err := certTable.Unmarshal(certs); err != nil { + return nil, nil, fmt.Errorf("unmarshalling SNP certificate table: %w", err) + } + + vcekRaw, err := certTable.GetByGUIDString(abi.VcekGUID) + if err != nil { + return nil, nil, fmt.Errorf("getting VCEK certificate: %w", err) + } + + // An optional check for certificate well-formedness. vcekRaw == cert.Raw. + vcek, err := x509.ParseCertificate(vcekRaw) + if err != nil { + return nil, nil, fmt.Errorf("parsing certificate: %w", err) + } + + vcekPEM = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: vcek.Raw, + }) + + var askPEM []byte + if askRaw, err := certTable.GetByGUIDString(abi.AskGUID); err == nil { + ask, err := x509.ParseCertificate(askRaw) + if err != nil { + return nil, nil, fmt.Errorf("parsing ASK certificate: %w", err) + } + + askPEM = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: ask.Raw, + }) + } + + return vcekPEM, askPEM, nil +} diff --git a/internal/attestation/gcp/snp/snp.go b/internal/attestation/gcp/snp/snp.go new file mode 100644 index 000000000..d1e42728f --- /dev/null +++ b/internal/attestation/gcp/snp/snp.go @@ -0,0 +1,42 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +/* +# GCP SEV-SNP attestation + +Google offers [confidential VMs], utilizing AMD SEV-SNP to provide memory encryption. + +Each SEV-SNP VM comes with a [virtual Trusted Platform Module (vTPM)]. +This vTPM can be used to generate encryption keys unique to the VM or to attest the platform's boot chain. +We can use the vTPM to verify the VM is running on AMD SEV-SNP enabled hardware and booted the expected OS image, allowing us to bootstrap a constellation cluster. + +# Issuer + +Retrieves an SEV-SNP attestation statement for the VM it's running in. Then, it generates a TPM attestation statement, binding the SEV-SNP attestation statement to it by including its hash in the TPM attestation statement. +Without binding the SEV-SNP attestation statement to the TPM attestation statement, the SEV-SNP attestation statement could be used in a different VM. Furthermore, it's important to first create the SEV-SNP attestation statement +and then the TPM attestation statement, as otherwise, a non-CVM could be used to create a valid TPM attestation statement, and then later swap the SEV-SNP attestation statement with one from a CVM. +Additionally project ID, zone, and instance name are fetched from the metadata server and attached to the attestation statement. + +# Validator + +First, it verifies the SEV-SNP attestation statement by checking the signatures and claims. Then, it verifies the TPM attestation by using a +public key provided by Google's API corresponding to the project ID, zone, instance name tuple attached to the attestation document, and confirms whether the SEV-SNP attestation statement is bound to the TPM attestation statement. + +# Problems + + - We have to trust Google + + Since the vTPM is provided by Google, and they could do whatever they want with it, we have no save proof of the VMs actually being confidential. + + - The provided vTPM has no endorsement certificate for its attestation key + + Without a certificate signing the authenticity of any endorsement keys we have no way of establishing a chain of trust. + Instead, we have to rely on Google's API to provide us with the public key of the vTPM's endorsement key. + +[confidential VMs]: https://cloud.google.com/compute/confidential-vm/docs/about-cvm +[virtual Trusted Platform Module (vTPM)]: https://cloud.google.com/security/shielded-cloud/shielded-vm#vtpm +*/ +package snp diff --git a/internal/attestation/gcp/snp/validator.go b/internal/attestation/gcp/snp/validator.go new file mode 100644 index 000000000..2dffff7b6 --- /dev/null +++ b/internal/attestation/gcp/snp/validator.go @@ -0,0 +1,206 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package snp + +import ( + "context" + "crypto" + "crypto/x509" + "encoding/json" + "fmt" + + "github.com/edgelesssys/constellation/v2/internal/attestation" + "github.com/edgelesssys/constellation/v2/internal/attestation/gcp" + "github.com/edgelesssys/constellation/v2/internal/attestation/snp" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/google/go-sev-guest/abi" + "github.com/google/go-sev-guest/kds" + "github.com/google/go-sev-guest/proto/sevsnp" + "github.com/google/go-sev-guest/validate" + "github.com/google/go-sev-guest/verify" + "github.com/google/go-sev-guest/verify/trust" + "github.com/google/go-tpm-tools/proto/attest" +) + +// Validator for GCP SEV-SNP / TPM attestation. +type Validator struct { + variant.GCPSEVSNP + *vtpm.Validator + cfg *config.GCPSEVSNP + + // reportValidator validates a SNP report and is required for testing. + reportValidator snpReportValidator + + // gceKeyGetter gets the public key of the EK from the GCE metadata API. + gceKeyGetter func(ctx context.Context, attDoc vtpm.AttestationDocument, _ []byte) (crypto.PublicKey, error) + + log attestation.Logger +} + +// NewValidator creates a new Validator. +func NewValidator(cfg *config.GCPSEVSNP, log attestation.Logger) (*Validator, error) { + getGCEKey, err := gcp.TrustedKeyGetter(variant.GCPSEVSNP{}, gcp.NewRESTClient) + if err != nil { + return nil, fmt.Errorf("creating trusted key getter: %w", err) + } + + v := &Validator{ + cfg: cfg, + reportValidator: &gcpValidator{httpsGetter: trust.DefaultHTTPSGetter(), verifier: &reportVerifierImpl{}, validator: &reportValidatorImpl{}}, + gceKeyGetter: getGCEKey, + log: log, + } + + v.Validator = vtpm.NewValidator( + cfg.Measurements, + v.getTrustedKey, + func(_ vtpm.AttestationDocument, _ *attest.MachineState) error { return nil }, + log, + ) + return v, nil +} + +// getTrustedKey returns TPM endorsement key provided through the GCE metadata API. +func (v *Validator) getTrustedKey(ctx context.Context, attDoc vtpm.AttestationDocument, extraData []byte) (crypto.PublicKey, error) { + if len(extraData) > 64 { + return nil, fmt.Errorf("extra data too long: %d, should be 64 bytes at most", len(extraData)) + } + var extraData64 [64]byte + copy(extraData64[:], extraData) + + if err := v.reportValidator.validate(attDoc, (*x509.Certificate)(&v.cfg.AMDSigningKey), (*x509.Certificate)(&v.cfg.AMDRootKey), extraData64, v.cfg, v.log); err != nil { + return nil, fmt.Errorf("validating SNP report: %w", err) + } + + ekPub, err := v.gceKeyGetter(ctx, attDoc, nil) + if err != nil { + return nil, fmt.Errorf("getting TPM endorsement key: %w", err) + } + + return ekPub, nil +} + +// snpReportValidator validates a given SNP report. +type snpReportValidator interface { + validate(attestation vtpm.AttestationDocument, ask *x509.Certificate, ark *x509.Certificate, ak [64]byte, config *config.GCPSEVSNP, log attestation.Logger) error +} + +// gcpValidator implements the validation for GCP SEV-SNP attestation. +// The properties exist for unittesting. +type gcpValidator struct { + verifier reportVerifier + validator reportValidator + httpsGetter trust.HTTPSGetter +} + +type reportVerifier interface { + SnpAttestation(att *sevsnp.Attestation, opts *verify.Options) error +} +type reportValidator interface { + SnpAttestation(att *sevsnp.Attestation, opts *validate.Options) error +} + +type reportValidatorImpl struct{} + +func (r *reportValidatorImpl) SnpAttestation(att *sevsnp.Attestation, opts *validate.Options) error { + return validate.SnpAttestation(att, opts) +} + +type reportVerifierImpl struct{} + +func (r *reportVerifierImpl) SnpAttestation(att *sevsnp.Attestation, opts *verify.Options) error { + return verify.SnpAttestation(att, opts) +} + +// validate the report by checking if it has a valid VCEK signature. +// The certificate chain ARK -> ASK -> VCEK is also validated. +// Checks that the report's userData matches the connection's userData. +func (a *gcpValidator) validate(attestation vtpm.AttestationDocument, ask *x509.Certificate, ark *x509.Certificate, reportData [64]byte, config *config.GCPSEVSNP, log attestation.Logger) error { + var info snp.InstanceInfo + if err := json.Unmarshal(attestation.InstanceInfo, &info); err != nil { + return fmt.Errorf("unmarshalling instance info: %w", err) + } + + certchain := snp.NewCertificateChain(ask, ark) + + att, err := info.AttestationWithCerts(a.httpsGetter, certchain, log) + if err != nil { + return fmt.Errorf("getting attestation with certs: %w", err) + } + + verifyOpts, err := getVerifyOpts(att) + if err != nil { + return fmt.Errorf("getting verify options: %w", err) + } + + if err := a.verifier.SnpAttestation(att, verifyOpts); err != nil { + return fmt.Errorf("verifying SNP attestation: %w", err) + } + + validateOpts := &validate.Options{ + // Check that the attestation key's digest is included in the report. + ReportData: reportData[:], + GuestPolicy: abi.SnpPolicy{ + Debug: false, // Debug means the VM can be decrypted by the host for debugging purposes and thus is not allowed. + SMT: true, // Allow Simultaneous Multi-Threading (SMT). Normally, we would want to disable SMT + // but GCP machines are currently facing issues if it's disabled + }, + VMPL: new(int), // Checks that Virtual Machine Privilege Level (VMPL) is 0. + // This checks that the reported LaunchTCB version is equal or greater than the minimum specified in the config. + // We don't specify Options.MinimumTCB as it only restricts the allowed TCB for Current_ and Reported_TCB. + // Because we allow Options.ProvisionalFirmware, there is not security gained in also checking Current_ and Reported_TCB. + // We always have to check Launch_TCB as this value indicated the smallest TCB version a VM has seen during + // it's lifetime. + MinimumLaunchTCB: kds.TCBParts{ + BlSpl: config.BootloaderVersion.Value, // Bootloader + TeeSpl: config.TEEVersion.Value, // TEE (Secure OS) + SnpSpl: config.SNPVersion.Value, // SNP + UcodeSpl: config.MicrocodeVersion.Value, // Microcode + }, + // Check that CurrentTCB >= CommittedTCB. + PermitProvisionalFirmware: true, + } + + // Checks if the attestation report matches the given constraints. + // Some constraints are implicitly checked by validate.SnpAttestation: + // - the report is not expired + if err := a.validator.SnpAttestation(att, validateOpts); err != nil { + return fmt.Errorf("validating SNP attestation: %w", err) + } + + return nil +} + +func getVerifyOpts(att *sevsnp.Attestation) (*verify.Options, error) { + ask, err := x509.ParseCertificate(att.CertificateChain.AskCert) + if err != nil { + return nil, fmt.Errorf("parsing ASK certificate: %w", err) + } + ark, err := x509.ParseCertificate(att.CertificateChain.ArkCert) + if err != nil { + return nil, fmt.Errorf("parsing ARK certificate: %w", err) + } + + verifyOpts := &verify.Options{ + DisableCertFetching: true, + TrustedRoots: map[string][]*trust.AMDRootCerts{ + "Milan": { + { + Product: "Milan", + ProductCerts: &trust.ProductCerts{ + Ask: ask, + Ark: ark, + }, + }, + }, + }, + } + + return verifyOpts, nil +} diff --git a/internal/attestation/gcp/validator.go b/internal/attestation/gcp/validator.go deleted file mode 100644 index 310a33b55..000000000 --- a/internal/attestation/gcp/validator.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "crypto" - "crypto/x509" - "encoding/json" - "encoding/pem" - "fmt" - - compute "cloud.google.com/go/compute/apiv1" - "cloud.google.com/go/compute/apiv1/computepb" - "github.com/edgelesssys/constellation/v2/internal/attestation" - "github.com/edgelesssys/constellation/v2/internal/attestation/variant" - "github.com/edgelesssys/constellation/v2/internal/attestation/vtpm" - "github.com/edgelesssys/constellation/v2/internal/config" - "github.com/google/go-tpm-tools/proto/attest" - "github.com/googleapis/gax-go/v2" - "google.golang.org/api/option" -) - -const minimumGceVersion = 1 - -// Validator for GCP confidential VM attestation. -type Validator struct { - variant.GCPSEVES - *vtpm.Validator - - restClient func(context.Context, ...option.ClientOption) (gcpRestClient, error) -} - -// NewValidator initializes a new GCP validator with the provided PCR values. -func NewValidator(cfg *config.GCPSEVES, log attestation.Logger) *Validator { - v := &Validator{ - restClient: newInstanceClient, - } - v.Validator = vtpm.NewValidator( - cfg.Measurements, - v.trustedKeyFromGCEAPI, - validateCVM, - log, - ) - - return v -} - -type gcpRestClient interface { - GetShieldedInstanceIdentity(ctx context.Context, req *computepb.GetShieldedInstanceIdentityInstanceRequest, opts ...gax.CallOption) (*computepb.ShieldedInstanceIdentity, error) - Close() error -} - -type instanceClient struct { - *compute.InstancesClient -} - -func newInstanceClient(ctx context.Context, opts ...option.ClientOption) (gcpRestClient, error) { - c, err := compute.NewInstancesRESTClient(ctx, opts...) - if err != nil { - return nil, err - } - return &instanceClient{c}, nil -} - -// trustedKeyFromGCEAPI queries the GCE API for a shieldedVM's public signing key. -// This key can be used to verify attestation statements issued by the VM. -func (v *Validator) trustedKeyFromGCEAPI(ctx context.Context, attDoc vtpm.AttestationDocument, _ []byte) (crypto.PublicKey, error) { - client, err := v.restClient(ctx) - if err != nil { - return nil, fmt.Errorf("creating GCE client: %w", err) - } - defer client.Close() - - var instanceInfo attest.GCEInstanceInfo - if err := json.Unmarshal(attDoc.InstanceInfo, &instanceInfo); err != nil { - return nil, err - } - - instance, err := client.GetShieldedInstanceIdentity(ctx, &computepb.GetShieldedInstanceIdentityInstanceRequest{ - Instance: instanceInfo.GetInstanceName(), - Project: instanceInfo.GetProjectId(), - Zone: instanceInfo.GetZone(), - }) - if err != nil { - return nil, fmt.Errorf("retrieving VM identity: %w", err) - } - - if instance.SigningKey == nil || instance.SigningKey.EkPub == nil { - return nil, fmt.Errorf("received no signing key from GCP API") - } - - // Parse the signing key return by GetShieldedInstanceIdentity - block, _ := pem.Decode([]byte(*instance.SigningKey.EkPub)) - if block == nil || block.Type != "PUBLIC KEY" { - return nil, fmt.Errorf("failed to decode PEM block containing public key") - } - - return x509.ParsePKIXPublicKey(block.Bytes) -} - -// validateCVM checks that the machine state represents a GCE AMD-SEV VM. -func validateCVM(_ vtpm.AttestationDocument, state *attest.MachineState) error { - gceVersion := state.Platform.GetGceVersion() - if gceVersion < minimumGceVersion { - return fmt.Errorf("outdated GCE version: %v (require >= %v)", gceVersion, minimumGceVersion) - } - - tech := state.Platform.Technology - wantTech := attest.GCEConfidentialTechnology_AMD_SEV - if tech != wantTech { - return fmt.Errorf("unexpected confidential technology: %v (expected: %v)", tech, wantTech) - } - - return nil -} diff --git a/internal/attestation/idkeydigest/idkeydigest.go b/internal/attestation/idkeydigest/idkeydigest.go index 45b5a54c0..4e97c6ef0 100644 --- a/internal/attestation/idkeydigest/idkeydigest.go +++ b/internal/attestation/idkeydigest/idkeydigest.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package idkeydigest provides type definitions for the `idkeydigest` value of SEV-SNP attestation. diff --git a/internal/attestation/idkeydigest/idkeydigest_test.go b/internal/attestation/idkeydigest/idkeydigest_test.go index 9f83b20e7..85ea0bf73 100644 --- a/internal/attestation/idkeydigest/idkeydigest_test.go +++ b/internal/attestation/idkeydigest/idkeydigest_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package idkeydigest diff --git a/internal/attestation/initialize/initialize.go b/internal/attestation/initialize/initialize.go index 65bfe349c..5916e6a87 100644 --- a/internal/attestation/initialize/initialize.go +++ b/internal/attestation/initialize/initialize.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package initialize implements functions to mark a node as initialized in the context of cluster attestation. diff --git a/internal/attestation/initialize/initialize_test.go b/internal/attestation/initialize/initialize_test.go index bd31e60f0..a3e386fd4 100644 --- a/internal/attestation/initialize/initialize_test.go +++ b/internal/attestation/initialize/initialize_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package initialize diff --git a/internal/attestation/measurements/BUILD.bazel b/internal/attestation/measurements/BUILD.bazel index b3b615e19..ea3fe108b 100644 --- a/internal/attestation/measurements/BUILD.bazel +++ b/internal/attestation/measurements/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "measurements_enterprise.go", # keep "measurements_oss.go", + "overrides.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/attestation/measurements", visibility = ["//:__subpackages__"], diff --git a/internal/attestation/measurements/fetchmeasurements.go b/internal/attestation/measurements/fetchmeasurements.go index 7720d1a59..4c7d1026a 100644 --- a/internal/attestation/measurements/fetchmeasurements.go +++ b/internal/attestation/measurements/fetchmeasurements.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measurements diff --git a/internal/attestation/measurements/fetchmeasurements_test.go b/internal/attestation/measurements/fetchmeasurements_test.go index d79a77a41..e9af9edb2 100644 --- a/internal/attestation/measurements/fetchmeasurements_test.go +++ b/internal/attestation/measurements/fetchmeasurements_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measurements @@ -141,7 +141,7 @@ func TestFetchMeasurements(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) sut := NewVerifyFetcher(tc.cosign, tc.rekor, client) - m, err := sut.FetchAndVerifyMeasurements(context.Background(), "v999.999.999", cloudprovider.GCP, variant.GCPSEVES{}, tc.noVerify) + m, err := sut.FetchAndVerifyMeasurements(t.Context(), "v999.999.999", cloudprovider.GCP, variant.GCPSEVES{}, tc.noVerify) if tc.wantErr { assert.Error(err) if tc.asRekorErr { diff --git a/internal/attestation/measurements/measurement-generator/BUILD.bazel b/internal/attestation/measurements/measurement-generator/BUILD.bazel index 35e8c8259..a678991e9 100644 --- a/internal/attestation/measurements/measurement-generator/BUILD.bazel +++ b/internal/attestation/measurements/measurement-generator/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//internal/attestation/measurements", "//internal/attestation/variant", "//internal/cloud/cloudprovider", + "//internal/constants", "//internal/sigstore", "//internal/sigstore/keyselect", "@org_golang_x_tools//go/ast/astutil", diff --git a/internal/attestation/measurements/measurement-generator/generate.go b/internal/attestation/measurements/measurement-generator/generate.go index 0050760ad..f5c71a8e4 100644 --- a/internal/attestation/measurements/measurement-generator/generate.go +++ b/internal/attestation/measurements/measurement-generator/generate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -27,6 +27,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/sigstore" "github.com/edgelesssys/constellation/v2/internal/sigstore/keyselect" "golang.org/x/tools/go/ast/astutil" @@ -83,9 +84,9 @@ func main() { log.Println("Found", variant) returnStmtCtr++ // retrieve and validate measurements for the given CSP and image - measuremnts := mustGetMeasurements(ctx, rekor, provider, variant, defaultImage) + measurements := mustGetMeasurements(ctx, rekor, provider, variant, defaultImage) // replace the return statement with a composite literal containing the validated measurements - clause.Values[0] = measurementsCompositeLiteral(measuremnts) + clause.Values[0] = measurementsCompositeLiteral(measurements) } return true }, nil, @@ -109,11 +110,11 @@ func main() { // mustGetMeasurements fetches the measurements for the given image and CSP and verifies them. func mustGetMeasurements(ctx context.Context, verifier rekorVerifier, provider cloudprovider.Provider, attestationVariant variant.Variant, image string) measurements.M { - measurementsURL, err := measurementURL(image, "measurements.json") + measurementsURL, err := measurementURL(image, constants.CDNMeasurementsFile) if err != nil { panic(err) } - signatureURL, err := measurementURL(image, "measurements.json.sig") + signatureURL, err := measurementURL(image, constants.CDNMeasurementsSignature) if err != nil { panic(err) } @@ -266,8 +267,12 @@ func attestationVariantFromGoIdentifier(identifier string) (variant.Variant, err return variant.AWSNitroTPM{}, nil case "GCPSEVES": return variant.GCPSEVES{}, nil + case "GCPSEVSNP": + return variant.GCPSEVSNP{}, nil case "AzureSEVSNP": return variant.AzureSEVSNP{}, nil + case "AzureTDX": + return variant.AzureTDX{}, nil case "AzureTrustedLaunch": return variant.AzureTrustedLaunch{}, nil case "QEMUVTPM": diff --git a/internal/attestation/measurements/measurement-generator/generate_test.go b/internal/attestation/measurements/measurement-generator/generate_test.go index d030a9138..c7ea5df5c 100644 --- a/internal/attestation/measurements/measurement-generator/generate_test.go +++ b/internal/attestation/measurements/measurement-generator/generate_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package main @@ -13,5 +13,5 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } diff --git a/internal/attestation/measurements/measurements.go b/internal/attestation/measurements/measurements.go index 6e775d06b..13a881996 100644 --- a/internal/attestation/measurements/measurements.go +++ b/internal/attestation/measurements/measurements.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -26,6 +26,7 @@ import ( "net/url" "sort" "strconv" + "strings" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" @@ -330,6 +331,15 @@ func (m *M) UnmarshalYAML(unmarshal func(any) error) error { return nil } +// String returns a string representation of the measurements. +func (m M) String() string { + var returnString string + for i, measurement := range m { + returnString = strings.Join([]string{returnString, fmt.Sprintf("%d: 0x%s", i, hex.EncodeToString(measurement.Expected))}, ",") + } + return returnString +} + func (m *M) fromImageMeasurementsV2( measurements ImageMeasurementsV2, wantVersion versionsapi.Version, csp cloudprovider.Provider, attestationVariant variant.Variant, @@ -470,17 +480,17 @@ func (m *Measurement) unmarshal(eM encodedMeasurement) error { // WithAllBytes returns a measurement value where all bytes are set to b. Takes a dynamic length as input. // Expected are either 32 bytes (PCRMeasurementLength) or 48 bytes (TDXMeasurementLength). // Over inputs are possible in this function, but potentially rejected elsewhere. -func WithAllBytes(b byte, validationOpt MeasurementValidationOption, len int) Measurement { +func WithAllBytes(b byte, validationOpt MeasurementValidationOption, length int) Measurement { return Measurement{ - Expected: bytes.Repeat([]byte{b}, len), + Expected: bytes.Repeat([]byte{b}, length), ValidationOpt: validationOpt, } } // PlaceHolderMeasurement returns a measurement with placeholder values for Expected. -func PlaceHolderMeasurement(len int) Measurement { +func PlaceHolderMeasurement(length int) Measurement { return Measurement{ - Expected: bytes.Repeat([]byte{0x12, 0x34}, len/2), + Expected: bytes.Repeat([]byte{0x12, 0x34}, length/2), ValidationOpt: Enforce, } } @@ -497,12 +507,21 @@ func DefaultsFor(provider cloudprovider.Provider, attestationVariant variant.Var case provider == cloudprovider.Azure && attestationVariant == variant.AzureSEVSNP{}: return azure_AzureSEVSNP.Copy() + case provider == cloudprovider.Azure && attestationVariant == variant.AzureTDX{}: + return azure_AzureTDX.Copy() + case provider == cloudprovider.Azure && attestationVariant == variant.AzureTrustedLaunch{}: return azure_AzureTrustedLaunch.Copy() case provider == cloudprovider.GCP && attestationVariant == variant.GCPSEVES{}: return gcp_GCPSEVES.Copy() + case provider == cloudprovider.GCP && attestationVariant == variant.GCPSEVSNP{}: + return gcp_GCPSEVSNP.Copy() + + case provider == cloudprovider.OpenStack && attestationVariant == variant.QEMUVTPM{}: + return openstack_QEMUVTPM.Copy() + case provider == cloudprovider.QEMU && attestationVariant == variant.QEMUTDX{}: return qemu_QEMUTDX.Copy() diff --git a/internal/attestation/measurements/measurements_enterprise.go b/internal/attestation/measurements/measurements_enterprise.go index bb47bf88b..210a654bc 100644 --- a/internal/attestation/measurements/measurements_enterprise.go +++ b/internal/attestation/measurements/measurements_enterprise.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measurements @@ -13,14 +13,20 @@ package measurements // a build tag. // The enterprise build tag is required to validate the measurements using production // sigstore certificates. +// +// To add measurements for a new variant, add a new entry as `_ = M{}` and run the generate tool. +// Entries defined as `_ M` are ignored. // revive:disable:var-naming var ( - aws_AWSNitroTPM = M{0: {Expected: []byte{0x73, 0x7f, 0x76, 0x7a, 0x12, 0xf5, 0x4e, 0x70, 0xee, 0xcb, 0xc8, 0x68, 0x40, 0x11, 0x32, 0x3a, 0xe2, 0xfe, 0x2d, 0xd9, 0xf9, 0x07, 0x85, 0x57, 0x79, 0x69, 0xd7, 0xa2, 0x01, 0x3e, 0x8c, 0x12}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x2f, 0xd4, 0x5c, 0x59, 0xae, 0xac, 0x2c, 0x64, 0x24, 0x5f, 0x6e, 0xd0, 0x05, 0x38, 0x22, 0x73, 0x66, 0x4d, 0x0e, 0x36, 0xd0, 0xec, 0xa7, 0xea, 0xec, 0xda, 0xcf, 0xac, 0x73, 0x07, 0x84, 0x8a}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x88, 0x46, 0x42, 0x2e, 0x06, 0x91, 0xb8, 0x00, 0xb0, 0xd6, 0x10, 0x55, 0x06, 0xb5, 0xc8, 0xc3, 0xe8, 0x0e, 0xe0, 0x82, 0x2a, 0x42, 0x29, 0xde, 0xf0, 0xed, 0xf9, 0x0c, 0x3f, 0xc1, 0x8f, 0xda}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x3a, 0x2d, 0x1b, 0xc7, 0x1b, 0xcb, 0x45, 0x02, 0x3a, 0x0f, 0xab, 0x63, 0x01, 0x4e, 0x78, 0x48, 0xfb, 0x06, 0x0e, 0x2a, 0xee, 0xe5, 0x49, 0xef, 0x8d, 0xa9, 0x20, 0x7b, 0x4a, 0x55, 0xc7, 0xf6}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} - aws_AWSSEVSNP = M{0: {Expected: []byte{0x7b, 0x06, 0x8c, 0x0c, 0x3a, 0xc2, 0x9a, 0xfe, 0x26, 0x41, 0x34, 0x53, 0x6b, 0x9b, 0xe2, 0x6f, 0x1d, 0x4c, 0xcd, 0x57, 0x5b, 0x88, 0xd3, 0xc3, 0xce, 0xab, 0xf3, 0x6a, 0xc9, 0x9c, 0x02, 0x78}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0xe7, 0xfc, 0x1b, 0x28, 0xe3, 0x00, 0x79, 0x46, 0x1a, 0x60, 0x3d, 0x71, 0x81, 0xa0, 0x4d, 0x4b, 0xc4, 0xbf, 0xad, 0x12, 0x08, 0x9b, 0x33, 0x82, 0x47, 0xd3, 0x59, 0xe8, 0x26, 0x1f, 0x97, 0xea}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0xf6, 0x8e, 0x1c, 0x36, 0xf1, 0x20, 0x60, 0xf3, 0x57, 0xa1, 0x0e, 0x04, 0xf5, 0xd4, 0xb7, 0x0d, 0x72, 0x70, 0xb6, 0xfc, 0x89, 0x45, 0xd0, 0xec, 0x26, 0xe5, 0x9f, 0x9d, 0x6f, 0xbd, 0xa3, 0x35}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x3a, 0x75, 0x39, 0xf5, 0x1c, 0x1b, 0xb5, 0x31, 0x0a, 0x40, 0x3c, 0x3d, 0x7e, 0xac, 0x88, 0x32, 0xef, 0x27, 0x14, 0x28, 0x36, 0xc6, 0xa3, 0x0b, 0x60, 0x24, 0xc6, 0xb2, 0x1e, 0x04, 0x89, 0x1c}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} - azure_AzureSEVSNP = M{1: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x2e, 0x67, 0x6e, 0xb7, 0xf8, 0x6c, 0xb9, 0xa3, 0x5b, 0x6d, 0x90, 0xbf, 0xe8, 0x9f, 0xf2, 0x7d, 0x3c, 0x8c, 0x7f, 0x59, 0xe5, 0x88, 0xac, 0xc3, 0x80, 0xc4, 0xb8, 0xee, 0x73, 0x9e, 0x13, 0x29}, ValidationOpt: Enforce}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x10, 0xc8, 0xc7, 0x4c, 0x84, 0x56, 0x67, 0xa6, 0x95, 0x0c, 0xf4, 0x9b, 0x5e, 0x27, 0x4f, 0x6c, 0x00, 0xc1, 0x03, 0x66, 0x54, 0xb7, 0x88, 0x1d, 0xf5, 0xe6, 0xf7, 0xcf, 0xf1, 0xcf, 0x29, 0xc5}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x3b, 0x30, 0xba, 0xae, 0xea, 0xd5, 0x97, 0x34, 0x48, 0xc3, 0x6d, 0xd1, 0x16, 0x68, 0x77, 0xa4, 0xb9, 0x9b, 0x52, 0x37, 0x59, 0x14, 0x25, 0x27, 0x6e, 0x74, 0xfc, 0xae, 0x4e, 0xcb, 0xcf, 0xb5}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + aws_AWSNitroTPM = M{0: {Expected: []byte{0x73, 0x7f, 0x76, 0x7a, 0x12, 0xf5, 0x4e, 0x70, 0xee, 0xcb, 0xc8, 0x68, 0x40, 0x11, 0x32, 0x3a, 0xe2, 0xfe, 0x2d, 0xd9, 0xf9, 0x07, 0x85, 0x57, 0x79, 0x69, 0xd7, 0xa2, 0x01, 0x3e, 0x8c, 0x12}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x65, 0xbf, 0xea, 0xea, 0x27, 0xa7, 0x72, 0xde, 0x9b, 0x91, 0x5d, 0x9e, 0x95, 0xe7, 0xd5, 0x2e, 0x4c, 0xe0, 0xf9, 0x47, 0x4e, 0x5f, 0x8f, 0x54, 0xd0, 0xe4, 0x1e, 0x6f, 0x51, 0xfd, 0xe1, 0x58}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x46, 0x75, 0x96, 0x2f, 0xff, 0x48, 0x3d, 0x43, 0x4a, 0x17, 0x28, 0x9b, 0x02, 0x9d, 0xb7, 0x9c, 0xee, 0x9b, 0x34, 0xc5, 0xdf, 0x4c, 0xc5, 0xbd, 0x38, 0x69, 0x94, 0x8f, 0x6c, 0x83, 0x3e, 0x09}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x15, 0xa7, 0xd9, 0xd4, 0x2d, 0xd7, 0xfd, 0x0b, 0x3d, 0x93, 0x70, 0xa4, 0xff, 0x75, 0x06, 0x24, 0x18, 0xb9, 0x28, 0x8b, 0x25, 0x1a, 0x1d, 0x6a, 0x88, 0x81, 0xf1, 0x84, 0xf6, 0x4e, 0x24, 0x30}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + aws_AWSSEVSNP = M{0: {Expected: []byte{0xd6, 0xdf, 0x85, 0x53, 0x58, 0xf5, 0xb1, 0x0f, 0x06, 0xf0, 0xfa, 0xb3, 0xf4, 0x08, 0xad, 0x26, 0xcd, 0x16, 0x5a, 0x29, 0x49, 0xba, 0xd6, 0x9e, 0x2c, 0xc7, 0x56, 0x92, 0x52, 0x9e, 0x66, 0x2a}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x3c, 0xc3, 0x03, 0xc6, 0x4f, 0x85, 0x5f, 0x01, 0x65, 0xf6, 0xf0, 0x94, 0x9f, 0xd8, 0x67, 0xf2, 0x32, 0x73, 0xc2, 0xe8, 0xa8, 0x6d, 0xb7, 0x28, 0x38, 0xf8, 0x98, 0x9e, 0x9a, 0x16, 0x31, 0x58}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x7e, 0xd4, 0xe2, 0x4a, 0xb5, 0x2f, 0x91, 0x2c, 0x5b, 0x13, 0x0a, 0xef, 0xef, 0x09, 0x56, 0x00, 0xbc, 0xf4, 0x6c, 0x5a, 0x98, 0xd5, 0x99, 0x10, 0xb4, 0x83, 0x76, 0xaa, 0x0b, 0x8d, 0x50, 0xbd}, ValidationOpt: Enforce}, 11: {Expected: []byte{0xfc, 0x6c, 0x8b, 0xa3, 0x44, 0x6f, 0x79, 0x33, 0x8e, 0x09, 0xce, 0xac, 0xeb, 0x80, 0xf9, 0x15, 0x0b, 0x4a, 0xe6, 0x17, 0x96, 0x3d, 0xcd, 0xb8, 0xe7, 0x9d, 0x90, 0x44, 0xe1, 0x4a, 0x52, 0x85}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + azure_AzureSEVSNP = M{1: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0xf7, 0xe1, 0x4d, 0xf7, 0xee, 0x08, 0xaf, 0xeb, 0x8b, 0x73, 0xb2, 0x90, 0x42, 0x68, 0xe6, 0xed, 0xe6, 0x5a, 0x3c, 0xbd, 0x62, 0x79, 0xb0, 0x0d, 0xa9, 0xb5, 0x5e, 0x0e, 0x4e, 0x73, 0x1a, 0x98}, ValidationOpt: Enforce}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x7c, 0xdf, 0xd4, 0xaa, 0xa5, 0x7f, 0x6b, 0xa9, 0xc3, 0xcb, 0xff, 0x3b, 0xe2, 0xc6, 0x70, 0xe2, 0x0f, 0x77, 0x6a, 0xea, 0xe0, 0x54, 0xf6, 0x63, 0xcf, 0x10, 0x58, 0x1b, 0xf0, 0x73, 0x0a, 0xaa}, ValidationOpt: Enforce}, 11: {Expected: []byte{0xa2, 0x8e, 0x42, 0x0a, 0xca, 0x11, 0xe5, 0x22, 0x00, 0xb9, 0xa8, 0x5b, 0x05, 0x03, 0x81, 0x48, 0x02, 0xf5, 0xda, 0x0d, 0xf3, 0xda, 0xdc, 0x9d, 0xb6, 0x69, 0xb3, 0x7c, 0x88, 0x51, 0xf8, 0x40}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + azure_AzureTDX = M{1: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0xee, 0x09, 0xc4, 0x03, 0x0c, 0xda, 0x6b, 0x15, 0x4c, 0x33, 0x34, 0xcc, 0x5f, 0xe8, 0xd6, 0x03, 0x66, 0xed, 0x08, 0x65, 0x37, 0x54, 0x6c, 0x89, 0xac, 0x30, 0xb7, 0x5c, 0x64, 0xdd, 0xad, 0x34}, ValidationOpt: Enforce}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x2c, 0xfa, 0x71, 0x11, 0xa6, 0x8f, 0xb6, 0xe4, 0x77, 0xbd, 0xf9, 0xeb, 0xc5, 0x03, 0x42, 0x83, 0x2c, 0x02, 0x45, 0xb6, 0xb8, 0x43, 0x76, 0x9e, 0x94, 0x43, 0xc2, 0x11, 0xae, 0x44, 0x1c, 0x0c}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x20, 0xc6, 0x2f, 0x57, 0x12, 0x6f, 0x01, 0xa9, 0xde, 0x29, 0xb1, 0x3a, 0x4b, 0x23, 0x73, 0x54, 0xee, 0x03, 0xf8, 0x5e, 0xf2, 0x92, 0xb6, 0x7c, 0x5a, 0x05, 0xaa, 0xd9, 0xeb, 0xd5, 0x0c, 0x6d}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} azure_AzureTrustedLaunch M - gcp_GCPSEVES = M{1: {Expected: []byte{0x74, 0x5f, 0x2f, 0xb4, 0x23, 0x5e, 0x46, 0x47, 0xaa, 0x0a, 0xd5, 0xac, 0xe7, 0x81, 0xcd, 0x92, 0x9e, 0xb6, 0x8c, 0x28, 0x87, 0x0e, 0x7d, 0xd5, 0xd1, 0xa1, 0x53, 0x58, 0x54, 0x32, 0x5e, 0x56}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x84, 0x13, 0x48, 0x6b, 0x82, 0x41, 0x03, 0x8f, 0xa3, 0x4c, 0xa5, 0x77, 0xc4, 0x4a, 0x7e, 0x2a, 0x48, 0x3f, 0x07, 0xd1, 0x92, 0x87, 0x25, 0x67, 0x8b, 0x65, 0x59, 0xf9, 0xa0, 0xbf, 0x92, 0xde}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x46, 0x22, 0x77, 0x89, 0x88, 0xb8, 0x57, 0x37, 0xa8, 0x8b, 0xad, 0x3b, 0x87, 0x4a, 0x7a, 0x5c, 0x41, 0x03, 0x29, 0xb7, 0xd6, 0x47, 0x16, 0xf1, 0x5e, 0x7a, 0x32, 0x44, 0x2a, 0x37, 0x16, 0xe0}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x6d, 0xde, 0x83, 0x87, 0x2d, 0xbc, 0x7a, 0x7a, 0x1b, 0x30, 0x35, 0x5d, 0xa7, 0x10, 0xbb, 0xea, 0x4c, 0xe9, 0x9e, 0x7a, 0xe5, 0x38, 0x92, 0x99, 0xce, 0x03, 0x3e, 0x8f, 0x73, 0xc3, 0xe6, 0xda}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + gcp_GCPSEVES = M{1: {Expected: []byte{0x36, 0x95, 0xdc, 0xc5, 0x5e, 0x3a, 0xa3, 0x40, 0x27, 0xc2, 0x77, 0x93, 0xc8, 0x5c, 0x72, 0x3c, 0x69, 0x7d, 0x70, 0x8c, 0x42, 0xd1, 0xf7, 0x3b, 0xd6, 0xfa, 0x4f, 0x26, 0x60, 0x8a, 0x5b, 0x24}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x3a, 0xec, 0xda, 0x76, 0xbf, 0xa0, 0x07, 0xc7, 0x5b, 0x10, 0x35, 0x50, 0x2c, 0x5d, 0x7a, 0xe6, 0xcc, 0xe0, 0x80, 0xd7, 0xe2, 0xb8, 0x00, 0x9c, 0xd8, 0x90, 0x9a, 0x7d, 0x87, 0xeb, 0x1c, 0x5a}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0xcc, 0x06, 0x30, 0x20, 0x3c, 0x92, 0x65, 0x55, 0x77, 0x3f, 0x88, 0x2d, 0x32, 0x29, 0xf6, 0xb5, 0xb3, 0xdc, 0xd1, 0xa0, 0xd2, 0x93, 0xfd, 0x41, 0xdd, 0xf4, 0xfd, 0xf0, 0x5a, 0x89, 0x36, 0x39}, ValidationOpt: Enforce}, 11: {Expected: []byte{0xcb, 0xcb, 0xf6, 0x3b, 0xc5, 0xf0, 0xf7, 0x84, 0xba, 0x2a, 0xe2, 0xfc, 0x87, 0xde, 0xa5, 0x95, 0xb0, 0xdd, 0x85, 0xae, 0x45, 0x95, 0xb0, 0xb9, 0x67, 0x3a, 0x30, 0xff, 0xd4, 0x5b, 0xbe, 0x42}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + gcp_GCPSEVSNP = M{1: {Expected: []byte{0x36, 0x95, 0xdc, 0xc5, 0x5e, 0x3a, 0xa3, 0x40, 0x27, 0xc2, 0x77, 0x93, 0xc8, 0x5c, 0x72, 0x3c, 0x69, 0x7d, 0x70, 0x8c, 0x42, 0xd1, 0xf7, 0x3b, 0xd6, 0xfa, 0x4f, 0x26, 0x60, 0x8a, 0x5b, 0x24}, ValidationOpt: WarnOnly}, 2: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 3: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 4: {Expected: []byte{0x1c, 0x21, 0x78, 0x82, 0x9b, 0x71, 0x04, 0x9a, 0xcf, 0x8b, 0x83, 0x6e, 0xe0, 0x35, 0x04, 0xa8, 0x00, 0x36, 0x56, 0xad, 0x6f, 0x2d, 0xf8, 0x4f, 0x2f, 0x5d, 0xde, 0x27, 0x03, 0xec, 0x10, 0x5b}, ValidationOpt: Enforce}, 6: {Expected: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}, ValidationOpt: WarnOnly}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0xad, 0xd3, 0x60, 0x79, 0xcd, 0x57, 0x84, 0x15, 0xa5, 0x48, 0xd2, 0x78, 0x2a, 0x7c, 0x10, 0x0f, 0x6c, 0xe4, 0x7c, 0xde, 0x43, 0xd8, 0x37, 0xf9, 0x07, 0xb6, 0x92, 0xb3, 0xbf, 0xec, 0x24, 0xb4}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x81, 0x8e, 0x84, 0xee, 0x6b, 0x1b, 0x0c, 0x1a, 0xea, 0xfe, 0xae, 0x21, 0x49, 0x21, 0x34, 0x43, 0x2d, 0xdf, 0x20, 0xbd, 0x08, 0xe0, 0x94, 0x68, 0xc3, 0xe0, 0xd7, 0xa3, 0x19, 0x9f, 0x27, 0x83}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + openstack_QEMUVTPM = M{4: {Expected: []byte{0xb0, 0x36, 0xf1, 0x26, 0x58, 0x82, 0x19, 0x07, 0xc1, 0x41, 0x98, 0x3c, 0x00, 0x45, 0xe8, 0xee, 0xc0, 0xc7, 0x1a, 0x58, 0x3c, 0x1f, 0x9b, 0x0f, 0xfe, 0xd7, 0xfd, 0x30, 0xdc, 0x84, 0xaa, 0x46}, ValidationOpt: Enforce}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x6a, 0x4a, 0xfe, 0x4f, 0x76, 0x1e, 0xef, 0xdd, 0x65, 0x3c, 0x3d, 0xc8, 0x62, 0x73, 0x17, 0xb5, 0xec, 0xda, 0xc2, 0x7f, 0x40, 0x3f, 0x1d, 0xfd, 0x54, 0xa1, 0x39, 0x4f, 0x39, 0x43, 0x70, 0xea}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x8c, 0xf1, 0x78, 0x9a, 0xe6, 0xe8, 0xb6, 0x01, 0xc6, 0x07, 0x61, 0x49, 0x6b, 0x8d, 0xc8, 0xaf, 0xda, 0xcc, 0xa1, 0xa9, 0x3e, 0xf1, 0xfa, 0x6c, 0x77, 0x29, 0x35, 0x3c, 0x13, 0xf4, 0x25, 0x1e}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} qemu_QEMUTDX M - qemu_QEMUVTPM = M{4: {Expected: []byte{0x18, 0xac, 0x38, 0xad, 0x3c, 0x0f, 0x51, 0xd2, 0x38, 0x3e, 0xbe, 0xda, 0x93, 0x99, 0xef, 0xc5, 0xfe, 0x27, 0xf6, 0x3d, 0xe9, 0xf7, 0x6d, 0x1d, 0x42, 0x49, 0xe4, 0x6f, 0x90, 0x46, 0x78, 0x7b}, ValidationOpt: Enforce}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x23, 0x3b, 0x66, 0x64, 0x16, 0x15, 0x14, 0x24, 0xae, 0x55, 0x4c, 0x58, 0xab, 0x88, 0xa7, 0xc9, 0x0a, 0xf8, 0x45, 0x31, 0x31, 0x11, 0x05, 0x06, 0x63, 0xf1, 0xac, 0xda, 0xc8, 0xd5, 0xe9, 0xfe}, ValidationOpt: Enforce}, 11: {Expected: []byte{0x7a, 0x95, 0xc8, 0xae, 0x5a, 0x94, 0x1e, 0xb7, 0xad, 0x8d, 0x1d, 0x46, 0x21, 0xd0, 0x48, 0x3e, 0x44, 0x92, 0x52, 0x7d, 0xe7, 0xe0, 0xf4, 0x07, 0xe1, 0x52, 0x96, 0x02, 0x17, 0x12, 0xeb, 0x74}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 14: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: WarnOnly}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} + qemu_QEMUVTPM = M{4: {Expected: []byte{0x8e, 0xce, 0xaf, 0x83, 0x1f, 0xed, 0x91, 0x03, 0x5a, 0x71, 0xca, 0xd9, 0x5a, 0x54, 0x78, 0x46, 0xbc, 0x91, 0xa4, 0x55, 0xde, 0xb4, 0x59, 0xaa, 0xd2, 0xe5, 0x53, 0x7b, 0x32, 0x53, 0x41, 0xbd}, ValidationOpt: Enforce}, 8: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 9: {Expected: []byte{0x61, 0x05, 0x4a, 0x64, 0x2e, 0x86, 0x25, 0x48, 0x4d, 0x22, 0xdd, 0x91, 0x50, 0x8b, 0x35, 0x27, 0x7e, 0x82, 0x9f, 0x52, 0x1a, 0x50, 0x93, 0xe3, 0xa7, 0x33, 0x14, 0x82, 0x51, 0x31, 0x49, 0xaa}, ValidationOpt: Enforce}, 11: {Expected: []byte{0xbd, 0x32, 0x39, 0xb7, 0x43, 0x9e, 0x3c, 0x85, 0x8f, 0x5a, 0x94, 0x49, 0xe1, 0x10, 0x6b, 0x66, 0x85, 0xc7, 0xa5, 0x5a, 0x79, 0x4a, 0xaf, 0x11, 0xb6, 0x68, 0x93, 0xae, 0x29, 0x2c, 0xde, 0x04}, ValidationOpt: Enforce}, 12: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 13: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}, 15: {Expected: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, ValidationOpt: Enforce}} ) diff --git a/internal/attestation/measurements/measurements_oss.go b/internal/attestation/measurements/measurements_oss.go index 9cb5f2a70..895a5d258 100644 --- a/internal/attestation/measurements/measurements_oss.go +++ b/internal/attestation/measurements/measurements_oss.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measurements @@ -19,7 +19,6 @@ var ( 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), } - aws_AWSSEVSNP = M{ 4: PlaceHolderMeasurement(PCRMeasurementLength), 8: WithAllBytes(0x00, Enforce, PCRMeasurementLength), @@ -29,7 +28,6 @@ var ( 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), } - azure_AzureSEVSNP = M{ 4: PlaceHolderMeasurement(PCRMeasurementLength), 8: WithAllBytes(0x00, Enforce, PCRMeasurementLength), @@ -39,6 +37,15 @@ var ( 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), } + azure_AzureTDX = M{ + 4: PlaceHolderMeasurement(PCRMeasurementLength), + 8: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + 9: PlaceHolderMeasurement(PCRMeasurementLength), + 11: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + 12: PlaceHolderMeasurement(PCRMeasurementLength), + 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), + } azure_AzureTrustedLaunch = M{ 4: PlaceHolderMeasurement(PCRMeasurementLength), 8: WithAllBytes(0x00, Enforce, PCRMeasurementLength), @@ -57,6 +64,24 @@ var ( 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), } + gcp_GCPSEVSNP = M{ + 4: PlaceHolderMeasurement(PCRMeasurementLength), + 8: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + 9: PlaceHolderMeasurement(PCRMeasurementLength), + 11: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + 12: PlaceHolderMeasurement(PCRMeasurementLength), + 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), + } + openstack_QEMUVTPM = M{ + 4: PlaceHolderMeasurement(PCRMeasurementLength), + 8: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + 9: PlaceHolderMeasurement(PCRMeasurementLength), + 11: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + 12: PlaceHolderMeasurement(PCRMeasurementLength), + 13: WithAllBytes(0x00, Enforce, PCRMeasurementLength), + uint32(PCRIndexClusterID): WithAllBytes(0x00, Enforce, PCRMeasurementLength), + } qemu_QEMUTDX = M{ 0: PlaceHolderMeasurement(TDXMeasurementLength), 1: PlaceHolderMeasurement(TDXMeasurementLength), diff --git a/internal/attestation/measurements/measurements_test.go b/internal/attestation/measurements/measurements_test.go index 73cee7479..f2cdf4d50 100644 --- a/internal/attestation/measurements/measurements_test.go +++ b/internal/attestation/measurements/measurements_test.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package measurements import ( "bytes" - "context" "encoding/json" "io" "net/http" @@ -458,7 +457,7 @@ func TestMeasurementsFetchAndVerify(t *testing.T) { require.NoError(err) hash, err := m.fetchAndVerify( - context.Background(), client, verifier, + t.Context(), client, verifier, measurementsURL, signatureURL, tc.imageVersion, tc.csp, diff --git a/internal/attestation/measurements/overrides.go b/internal/attestation/measurements/overrides.go new file mode 100644 index 000000000..d81851449 --- /dev/null +++ b/internal/attestation/measurements/overrides.go @@ -0,0 +1,134 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package measurements + +import ( + "fmt" + + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" +) + +var measurementOverridesForCSP = map[string]measurementOverride{ + cloudprovider.AWS.String(): { + MustEnforce: []uint32{ + 4, 8, 9, 11, 12, 13, uint32(PCRIndexClusterID), + }, + MustWarn: []uint32{ + 0, 2, 3, 6, 14, + }, + ValueOverrides: []valueOverride{ + {Index: 2, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 3, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 6, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 14, Value: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, + }, + }, + cloudprovider.Azure.String(): { + MustEnforce: []uint32{ + 4, 8, 9, 11, 12, 13, uint32(PCRIndexClusterID), + }, + MustWarn: []uint32{ + 1, 2, 3, 14, + }, + ValueOverrides: []valueOverride{ + {Index: 1, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 2, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 3, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 14, Value: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, + }, + }, + cloudprovider.GCP.String(): { + MustEnforce: []uint32{ + 4, 8, 9, 11, 12, 13, uint32(PCRIndexClusterID), + }, + MustWarn: []uint32{ + 1, 2, 3, 6, 14, + }, + ValueOverrides: []valueOverride{ + {Index: 1, Value: []byte{0x36, 0x95, 0xdc, 0xc5, 0x5e, 0x3a, 0xa3, 0x40, 0x27, 0xc2, 0x77, 0x93, 0xc8, 0x5c, 0x72, 0x3c, 0x69, 0x7d, 0x70, 0x8c, 0x42, 0xd1, 0xf7, 0x3b, 0xd6, 0xfa, 0x4f, 0x26, 0x60, 0x8a, 0x5b, 0x24}}, + {Index: 2, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 3, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 6, Value: []byte{0x3d, 0x45, 0x8c, 0xfe, 0x55, 0xcc, 0x03, 0xea, 0x1f, 0x44, 0x3f, 0x15, 0x62, 0xbe, 0xec, 0x8d, 0xf5, 0x1c, 0x75, 0xe1, 0x4a, 0x9f, 0xcf, 0x9a, 0x72, 0x34, 0xa1, 0x3f, 0x19, 0x8e, 0x79, 0x69}}, + {Index: 14, Value: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, + }, + }, + cloudprovider.OpenStack.String(): { + MustEnforce: []uint32{ + 4, 8, 9, 11, 12, 13, uint32(PCRIndexClusterID), + }, + MustWarn: []uint32{ + 14, + }, + ValueOverrides: []valueOverride{ + {Index: 14, Value: []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, + }, + }, +} + +var measurementOverridesForAttestationVariant = map[string]measurementOverride{ + variant.AWSNitroTPM{}.String(): { + ValueOverrides: []valueOverride{ + {Index: 0, Value: []byte{0x73, 0x7f, 0x76, 0x7a, 0x12, 0xf5, 0x4e, 0x70, 0xee, 0xcb, 0xc8, 0x68, 0x40, 0x11, 0x32, 0x3a, 0xe2, 0xfe, 0x2d, 0xd9, 0xf9, 0x07, 0x85, 0x57, 0x79, 0x69, 0xd7, 0xa2, 0x01, 0x3e, 0x8c, 0x12}}, + }, + }, + variant.AWSSEVSNP{}.String(): { + ValueOverrides: []valueOverride{ + {Index: 0, Value: []byte{0xd6, 0xdf, 0x85, 0x53, 0x58, 0xf5, 0xb1, 0x0f, 0x06, 0xf0, 0xfa, 0xb3, 0xf4, 0x08, 0xad, 0x26, 0xcd, 0x16, 0x5a, 0x29, 0x49, 0xba, 0xd6, 0x9e, 0x2c, 0xc7, 0x56, 0x92, 0x52, 0x9e, 0x66, 0x2a}}, + }, + }, +} + +type measurementOverride struct { + MustEnforce []uint32 + MustWarn []uint32 + ValueOverrides []valueOverride +} + +type valueOverride struct { + Index uint32 + Value []byte +} + +// ApplyOverrides applies overrides to the given measurements. +func ApplyOverrides(in M, csp cloudprovider.Provider, attestationVariant string) (M, error) { + out := in.Copy() + var matchingOverrides []measurementOverride + if attestationVariantOverride, ok := measurementOverridesForAttestationVariant[attestationVariant]; ok { + matchingOverrides = append(matchingOverrides, attestationVariantOverride) + } + if cspOverride, ok := measurementOverridesForCSP[csp.String()]; ok { + matchingOverrides = append(matchingOverrides, cspOverride) + } + for _, override := range matchingOverrides { + for _, i := range override.ValueOverrides { + m, ok := out[i.Index] + if !ok { + m = Measurement{} + } + m.Expected = i.Value + out[i.Index] = m + } + for _, i := range override.MustEnforce { + m, ok := out[i] + if !ok { + return nil, fmt.Errorf("missing measurement for PCR %d", i) + } + m.ValidationOpt = Enforce + out[i] = m + } + for _, i := range override.MustWarn { + m, ok := out[i] + if !ok { + return nil, fmt.Errorf("missing measurement for PCR %d", i) + } + m.ValidationOpt = WarnOnly + out[i] = m + } + } + return out, nil +} diff --git a/internal/attestation/qemu/issuer.go b/internal/attestation/qemu/issuer.go index 8214e8a27..bbc9b6c58 100644 --- a/internal/attestation/qemu/issuer.go +++ b/internal/attestation/qemu/issuer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package qemu diff --git a/internal/attestation/qemu/qemu.go b/internal/attestation/qemu/qemu.go index 424215a6e..1856470d4 100644 --- a/internal/attestation/qemu/qemu.go +++ b/internal/attestation/qemu/qemu.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/qemu/validator.go b/internal/attestation/qemu/validator.go index e2c172f3b..001acb3d7 100644 --- a/internal/attestation/qemu/validator.go +++ b/internal/attestation/qemu/validator.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package qemu diff --git a/internal/attestation/simulator/simulator.go b/internal/attestation/simulator/simulator.go index 03baabdf5..5e612445d 100644 --- a/internal/attestation/simulator/simulator.go +++ b/internal/attestation/simulator/simulator.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // TPM2 simulator used for unit tests. diff --git a/internal/attestation/simulator/simulator_disabled.go b/internal/attestation/simulator/simulator_disabled.go index 1470e20cd..61adff724 100644 --- a/internal/attestation/simulator/simulator_disabled.go +++ b/internal/attestation/simulator/simulator_disabled.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package simulator diff --git a/internal/attestation/snp/BUILD.bazel b/internal/attestation/snp/BUILD.bazel index 700a3aa86..5cca9028f 100644 --- a/internal/attestation/snp/BUILD.bazel +++ b/internal/attestation/snp/BUILD.bazel @@ -8,11 +8,12 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/attestation", - "//internal/constants", "@com_github_google_go_sev_guest//abi", + "@com_github_google_go_sev_guest//client", "@com_github_google_go_sev_guest//kds", "@com_github_google_go_sev_guest//proto/sevsnp", "@com_github_google_go_sev_guest//verify/trust", + "@com_github_google_go_tpm_tools//proto/attest", ], ) diff --git a/internal/attestation/snp/snp.go b/internal/attestation/snp/snp.go index e7b285ba6..a296d7721 100644 --- a/internal/attestation/snp/snp.go +++ b/internal/attestation/snp/snp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package SNP provides types shared by SNP-based attestation implementations. @@ -12,16 +12,20 @@ import ( "bytes" "crypto/x509" "encoding/pem" + "errors" "fmt" "github.com/edgelesssys/constellation/v2/internal/attestation" - "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/google/go-sev-guest/abi" + "github.com/google/go-sev-guest/client" "github.com/google/go-sev-guest/kds" spb "github.com/google/go-sev-guest/proto/sevsnp" "github.com/google/go-sev-guest/verify/trust" + "github.com/google/go-tpm-tools/proto/attest" ) +var errNoPemBlocks = errors.New("no PEM blocks found") + // Product returns the SEV product info currently supported by Constellation's SNP attestation. func Product() *spb.SevProduct { // sevProduct is the product info of the SEV platform as reported through CPUID[EAX=1]. @@ -29,6 +33,26 @@ func Product() *spb.SevProduct { return &spb.SevProduct{Name: spb.SevProduct_SEV_PRODUCT_MILAN, Stepping: 0} // Milan-B0 } +// GetExtendedReport retrieves the extended SNP report from the CVM. +func GetExtendedReport(reportData [64]byte) (report, certChain []byte, err error) { + qp, err := client.GetLeveledQuoteProvider() + if err != nil { + return nil, nil, fmt.Errorf("getting quote provider: %w", err) + } + quote, err := qp.GetRawQuoteAtLevel(reportData, 0) + if err != nil { + return nil, nil, fmt.Errorf("getting extended report: %w", err) + } + + // Parse the report and certificate chain from the quote. + report = quote + if len(quote) > abi.ReportSize { + report = quote[:abi.ReportSize] + certChain = quote[abi.ReportSize:] + } + return report, certChain, nil +} + // InstanceInfo contains the necessary information to establish trust in a SNP CVM. type InstanceInfo struct { // ReportSigner is the PEM-encoded certificate used to validate the attestation report's signature. @@ -39,6 +63,7 @@ type InstanceInfo struct { // AttestationReport is the attestation report from the vTPM (NVRAM) of the CVM. AttestationReport []byte Azure *AzureInstanceInfo + GCP *attest.GCEInstanceInfo } // AzureInstanceInfo contains Azure specific information related to SNP attestation. @@ -57,7 +82,7 @@ func (a *InstanceInfo) addReportSigner(att *spb.Attestation, report *spb.Report, // If the VCEK certificate is present, parse it and format it. reportSigner, err := a.ParseReportSigner() if err != nil { - logger.Warnf("Error parsing report signer: %v", err) + logger.Warn(fmt.Sprintf("Error parsing report signer: %v", err)) } signerInfo, err := abi.ParseSignerInfo(report.GetSignerInfo()) @@ -77,7 +102,7 @@ func (a *InstanceInfo) addReportSigner(att *spb.Attestation, report *spb.Report, // If no VCEK is present, fetch it from AMD. if reportSigner == nil { - logger.Infof("VCEK certificate not present, falling back to retrieving it from AMD KDS") + logger.Info("VCEK certificate not present, falling back to retrieving it from AMD KDS") vcekURL := kds.VCEKCertURL(productName, report.GetChipId(), kds.TCBVersion(report.GetReportedTcb())) vcekData, err = getter.Get(vcekURL) if err != nil { @@ -95,7 +120,7 @@ func (a *InstanceInfo) addReportSigner(att *spb.Attestation, report *spb.Report, // AttestationWithCerts returns a formatted version of the attestation report and its certificates from the instanceInfo. // Certificates are retrieved in the following precedence: -// 1. ASK or ARK from issuer. On Azure: THIM. One AWS: not prefilled. +// 1. ASK from issuer. On Azure: THIM. One AWS: not prefilled. (Go to option 2) On GCP: prefilled. // 2. ASK or ARK from fallbackCerts. // 3. ASK or ARK from AMD KDS. func (a *InstanceInfo) AttestationWithCerts(getter trust.HTTPSGetter, @@ -106,7 +131,7 @@ func (a *InstanceInfo) AttestationWithCerts(getter trust.HTTPSGetter, return nil, fmt.Errorf("converting report to proto: %w", err) } - productName := kds.ProductString(Product()) + productName := kds.ProductLine(Product()) att := &spb.Attestation{ Report: report, @@ -120,46 +145,44 @@ func (a *InstanceInfo) AttestationWithCerts(getter trust.HTTPSGetter, return nil, fmt.Errorf("adding report signer: %w", err) } - // If the certificate chain from THIM is present, parse it and format it. - ask, ark, err := a.ParseCertChain() - if err != nil { - logger.Warnf("Error parsing certificate chain: %v", err) + // If a certificate chain was pre-fetched by the Issuer, parse it and format it. + // Make sure to only use the ask, since using an ark from the Issuer would invalidate security guarantees. + ask, _, err := a.ParseCertChain() + if err != nil && !errors.Is(err, errNoPemBlocks) { + logger.Warn(fmt.Sprintf("Error parsing certificate chain: %v", err)) } if ask != nil { - logger.Infof("Using ASK certificate from Azure THIM") + logger.Info("Using ASK certificate from pre-fetched certificate chain") att.CertificateChain.AskCert = ask.Raw } - if ark != nil { - logger.Infof("Using ARK certificate from Azure THIM") - att.CertificateChain.ArkCert = ark.Raw - } // If a cached ASK or an ARK from the Constellation config is present, use it. if att.CertificateChain.AskCert == nil && fallbackCerts.ask != nil { - logger.Infof("Using cached ASK certificate") + logger.Info("Using cached ASK certificate") att.CertificateChain.AskCert = fallbackCerts.ask.Raw } - if att.CertificateChain.ArkCert == nil && fallbackCerts.ark != nil { - logger.Infof("Using ARK certificate from %s", constants.ConfigFilename) + if fallbackCerts.ark != nil { + logger.Info("Using cached ARK certificate") att.CertificateChain.ArkCert = fallbackCerts.ark.Raw } - // Otherwise, retrieve it from AMD KDS. + + // Otherwise, retrieve missing certificates from AMD KDS. if att.CertificateChain.AskCert == nil || att.CertificateChain.ArkCert == nil { - logger.Infof( + logger.Info(fmt.Sprintf( "Certificate chain not fully present (ARK present: %t, ASK present: %t), falling back to retrieving it from AMD KDS", (att.CertificateChain.ArkCert != nil), (att.CertificateChain.AskCert != nil), - ) + )) kdsCertChain, err := trust.GetProductChain(productName, signingInfo, getter) if err != nil { return nil, fmt.Errorf("retrieving certificate chain from AMD KDS: %w", err) } if att.CertificateChain.AskCert == nil && kdsCertChain.Ask != nil { - logger.Infof("Using ASK certificate from AMD KDS") + logger.Info("Using ASK certificate from AMD KDS") att.CertificateChain.AskCert = kdsCertChain.Ask.Raw } if att.CertificateChain.ArkCert == nil && kdsCertChain.Ask != nil { - logger.Infof("Using ARK certificate from AMD KDS") + logger.Info("Using ARK certificate from AMD KDS") att.CertificateChain.ArkCert = kdsCertChain.Ark.Raw } } @@ -223,7 +246,7 @@ func (a *InstanceInfo) ParseCertChain() (ask, ark *x509.Certificate, retErr erro switch { case i == 1: - retErr = fmt.Errorf("no PEM blocks found") + retErr = errNoPemBlocks case len(rest) != 0: retErr = fmt.Errorf("remaining PEM block is not a valid certificate: %s", rest) } diff --git a/internal/attestation/snp/snp_test.go b/internal/attestation/snp/snp_test.go index 0179ac05b..19485e47d 100644 --- a/internal/attestation/snp/snp_test.go +++ b/internal/attestation/snp/snp_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package snp @@ -9,6 +9,7 @@ package snp import ( "crypto/x509" "encoding/hex" + "errors" "fmt" "regexp" "strings" @@ -34,16 +35,13 @@ func TestParseCertChain(t *testing.T) { wantAsk bool wantArk bool wantErr bool + errTarget error }{ "success": { certChain: defaultCertChain, wantAsk: true, wantArk: true, }, - "empty cert chain": { - certChain: []byte{}, - wantErr: true, - }, "more than two certificates": { certChain: append(defaultCertChain, defaultCertChain...), wantErr: true, @@ -52,6 +50,11 @@ func TestParseCertChain(t *testing.T) { certChain: []byte("invalid"), wantErr: true, }, + "empty cert chain": { + certChain: []byte{}, + wantErr: true, + errTarget: errNoPemBlocks, + }, "ark missing": { certChain: []byte(askOnly), wantAsk: true, @@ -73,6 +76,9 @@ func TestParseCertChain(t *testing.T) { ask, ark, err := instanceInfo.ParseCertChain() if tc.wantErr { assert.Error(err) + if tc.errTarget != nil { + assert.True(errors.Is(err, tc.errTarget)) + } } else { assert.NoError(err) assert.Equal(tc.wantAsk, ask != nil) @@ -149,12 +155,24 @@ func TestAttestationWithCerts(t *testing.T) { wantErr bool }{ "success": { + report: defaultReport, + idkeydigest: "57e229e0ffe5fa92d0faddff6cae0e61c926fc9ef9afd20a8b8cfcf7129db9338cbe5bf3f6987733a2bf65d06dc38fc1", + reportSigner: testdata.AzureThimVCEK, + certChain: testdata.CertChain, + fallbackCerts: CertificateChain{ark: testdataArk}, + expectedArk: testdataArk, + expectedAsk: testdataAsk, + getter: newStubHTTPSGetter(&urlResponseMatcher{}, nil), + }, + "ark only in pre-fetched cert-chain": { report: defaultReport, idkeydigest: "57e229e0ffe5fa92d0faddff6cae0e61c926fc9ef9afd20a8b8cfcf7129db9338cbe5bf3f6987733a2bf65d06dc38fc1", reportSigner: testdata.AzureThimVCEK, certChain: testdata.CertChain, expectedArk: testdataArk, expectedAsk: testdataAsk, + getter: newStubHTTPSGetter(nil, assert.AnError), + wantErr: true, }, "vlek success": { report: vlekReport, @@ -173,9 +191,10 @@ func TestAttestationWithCerts(t *testing.T) { ), }, "retrieve vcek": { - report: defaultReport, - idkeydigest: "57e229e0ffe5fa92d0faddff6cae0e61c926fc9ef9afd20a8b8cfcf7129db9338cbe5bf3f6987733a2bf65d06dc38fc1", - certChain: testdata.CertChain, + report: defaultReport, + idkeydigest: "57e229e0ffe5fa92d0faddff6cae0e61c926fc9ef9afd20a8b8cfcf7129db9338cbe5bf3f6987733a2bf65d06dc38fc1", + certChain: testdata.CertChain, + fallbackCerts: CertificateChain{ark: testdataArk}, getter: newStubHTTPSGetter( &urlResponseMatcher{ vcekResponse: testdata.AmdKdsVCEK, @@ -205,25 +224,9 @@ func TestAttestationWithCerts(t *testing.T) { idkeydigest: "57e229e0ffe5fa92d0faddff6cae0e61c926fc9ef9afd20a8b8cfcf7129db9338cbe5bf3f6987733a2bf65d06dc38fc1", reportSigner: testdata.AzureThimVCEK, fallbackCerts: NewCertificateChain(exampleCert, exampleCert), - getter: newStubHTTPSGetter( - &urlResponseMatcher{}, - nil, - ), - expectedArk: exampleCert, - expectedAsk: exampleCert, - }, - "use certchain with fallback certs": { - report: defaultReport, - idkeydigest: "57e229e0ffe5fa92d0faddff6cae0e61c926fc9ef9afd20a8b8cfcf7129db9338cbe5bf3f6987733a2bf65d06dc38fc1", - certChain: testdata.CertChain, - reportSigner: testdata.AzureThimVCEK, - fallbackCerts: NewCertificateChain(&x509.Certificate{}, &x509.Certificate{}), - getter: newStubHTTPSGetter( - &urlResponseMatcher{}, - nil, - ), - expectedArk: testdataArk, - expectedAsk: testdataAsk, + getter: newStubHTTPSGetter(&urlResponseMatcher{}, nil), + expectedArk: exampleCert, + expectedAsk: exampleCert, }, "retrieve vcek and certchain": { report: defaultReport, @@ -242,10 +245,12 @@ func TestAttestationWithCerts(t *testing.T) { }, "report too short": { report: defaultReport[:len(defaultReport)-100], + getter: newStubHTTPSGetter(nil, assert.AnError), wantErr: true, }, "corrupted report": { report: defaultReport[10 : len(defaultReport)-10], + getter: newStubHTTPSGetter(nil, assert.AnError), wantErr: true, }, "certificate fetch error": { diff --git a/internal/attestation/snp/testdata/testdata.go b/internal/attestation/snp/testdata/testdata.go index c749dd899..ba93753bd 100644 --- a/internal/attestation/snp/testdata/testdata.go +++ b/internal/attestation/snp/testdata/testdata.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package testdata contains testing data for an attestation process. diff --git a/internal/attestation/tdx/issuer.go b/internal/attestation/tdx/issuer.go index 4bee70065..58be53de6 100644 --- a/internal/attestation/tdx/issuer.go +++ b/internal/attestation/tdx/issuer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package tdx @@ -37,10 +37,10 @@ func NewIssuer(log attestation.Logger) *Issuer { // Issue issues a TDX attestation document. func (i *Issuer) Issue(_ context.Context, userData []byte, nonce []byte) (attDoc []byte, err error) { - i.log.Infof("Issuing attestation statement") + i.log.Info("Issuing attestation statement") defer func() { if err != nil { - i.log.Warnf("Failed to issue attestation document: %s", err) + i.log.Warn(fmt.Sprintf("Failed to issue attestation document: %s", err)) } }() diff --git a/internal/attestation/tdx/tdx.go b/internal/attestation/tdx/tdx.go index ea0cb67c4..25141b609 100644 --- a/internal/attestation/tdx/tdx.go +++ b/internal/attestation/tdx/tdx.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package TDX implements attestation for Intel TDX. diff --git a/internal/attestation/tdx/validator.go b/internal/attestation/tdx/validator.go index 0a18f1c9d..6a5bde48a 100644 --- a/internal/attestation/tdx/validator.go +++ b/internal/attestation/tdx/validator.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package tdx @@ -49,10 +49,10 @@ func NewValidator(cfg *config.QEMUTDX, log attestation.Logger) *Validator { // Validate validates the given attestation document using TDX attestation. func (v *Validator) Validate(ctx context.Context, attDocRaw []byte, nonce []byte) (userData []byte, err error) { - v.log.Infof("Validating attestation document") + v.log.Info("Validating attestation document") defer func() { if err != nil { - v.log.Warnf("Failed to validate attestation document: %s", err) + v.log.Warn(fmt.Sprintf("Failed to validate attestation document: %s", err)) } }() @@ -83,7 +83,7 @@ func (v *Validator) Validate(ctx context.Context, attDocRaw []byte, nonce []byte // Verify the quote against the expected measurements. warnings, errs := v.expected.Compare(tdMeasure) for _, warning := range warnings { - v.log.Warnf(warning) + v.log.Warn(warning) } if len(errs) > 0 { return nil, fmt.Errorf("measurement validation failed:\n%w", errors.Join(errs...)) diff --git a/internal/attestation/variant/variant.go b/internal/attestation/variant/variant.go index 82fd1f1c9..97ed68244 100644 --- a/internal/attestation/variant/variant.go +++ b/internal/attestation/variant/variant.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -44,6 +44,8 @@ const ( awsNitroTPM = "aws-nitro-tpm" awsSEVSNP = "aws-sev-snp" gcpSEVES = "gcp-sev-es" + gcpSEVSNP = "gcp-sev-snp" + azureTDX = "azure-tdx" azureSEVSNP = "azure-sev-snp" azureTrustedLaunch = "azure-trustedlaunch" qemuVTPM = "qemu-vtpm" @@ -52,8 +54,8 @@ const ( var providerAttestationMapping = map[cloudprovider.Provider][]Variant{ cloudprovider.AWS: {AWSSEVSNP{}, AWSNitroTPM{}}, - cloudprovider.Azure: {AzureSEVSNP{}, AzureTrustedLaunch{}}, - cloudprovider.GCP: {GCPSEVES{}}, + cloudprovider.Azure: {AzureSEVSNP{}, AzureTDX{}, AzureTrustedLaunch{}}, + cloudprovider.GCP: {GCPSEVSNP{}, GCPSEVES{}}, cloudprovider.QEMU: {QEMUVTPM{}}, cloudprovider.OpenStack: {QEMUVTPM{}}, } @@ -109,10 +111,14 @@ func FromString(oid string) (Variant, error) { return AWSNitroTPM{}, nil case gcpSEVES: return GCPSEVES{}, nil + case gcpSEVSNP: + return GCPSEVSNP{}, nil case azureSEVSNP: return AzureSEVSNP{}, nil case azureTrustedLaunch: return AzureTrustedLaunch{}, nil + case azureTDX: + return AzureTDX{}, nil case qemuVTPM: return QEMUVTPM{}, nil case qemuTDX: @@ -206,6 +212,42 @@ func (GCPSEVES) Equal(other Getter) bool { return other.OID().Equal(GCPSEVES{}.OID()) } +// GCPSEVSNP holds the GCP SEV-SNP OID. +type GCPSEVSNP struct{} + +// OID returns the struct's object identifier. +func (GCPSEVSNP) OID() asn1.ObjectIdentifier { + return asn1.ObjectIdentifier{1, 3, 9900, 3, 2} +} + +// String returns the string representation of the OID. +func (GCPSEVSNP) String() string { + return gcpSEVSNP +} + +// Equal returns true if the other variant is also GCPSEVSNP. +func (GCPSEVSNP) Equal(other Getter) bool { + return other.OID().Equal(GCPSEVSNP{}.OID()) +} + +// AzureTDX holds the OID for Azure TDX CVMs. +type AzureTDX struct{} + +// OID returns the struct's object identifier. +func (AzureTDX) OID() asn1.ObjectIdentifier { + return asn1.ObjectIdentifier{1, 3, 9900, 4, 3} +} + +// String returns the string representation of the OID. +func (AzureTDX) String() string { + return azureTDX +} + +// Equal returns true if the other variant is also AzureTDX. +func (AzureTDX) Equal(other Getter) bool { + return other.OID().Equal(AzureTDX{}.OID()) +} + // AzureSEVSNP holds the OID for Azure SNP CVMs. type AzureSEVSNP struct{} diff --git a/internal/attestation/vtpm/attestation.go b/internal/attestation/vtpm/attestation.go index a8a5e9b0f..98d3a9ad2 100644 --- a/internal/attestation/vtpm/attestation.go +++ b/internal/attestation/vtpm/attestation.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package vtpm @@ -9,10 +9,12 @@ package vtpm import ( "context" "crypto" + "crypto/sha256" "encoding/json" "errors" "fmt" "io" + "slices" "github.com/google/go-sev-guest/proto/sevsnp" tpmClient "github.com/google/go-tpm-tools/client" @@ -103,10 +105,10 @@ func NewIssuer( // Issue generates an attestation document using a TPM. func (i *Issuer) Issue(ctx context.Context, userData []byte, nonce []byte) (res []byte, err error) { - i.log.Infof("Issuing attestation statement") + i.log.Info("Issuing attestation statement") defer func() { if err != nil { - i.log.Warnf("Failed to issue attestation statement: %s", err) + i.log.Warn(fmt.Sprintf("Failed to issue attestation statement: %s", err)) } }() @@ -123,12 +125,7 @@ func (i *Issuer) Issue(ctx context.Context, userData []byte, nonce []byte) (res } defer aK.Close() - // Create an attestation using the loaded key extraData := attestation.MakeExtraData(userData, nonce) - tpmAttestation, err := aK.Attest(tpmClient.AttestOpts{Nonce: extraData}) - if err != nil { - return nil, fmt.Errorf("creating attestation: %w", err) - } // Fetch instance info of the VM instanceInfo, err := i.getInstanceInfo(ctx, tpm, extraData) @@ -136,6 +133,14 @@ func (i *Issuer) Issue(ctx context.Context, userData []byte, nonce []byte) (res return nil, fmt.Errorf("fetching instance info: %w", err) } + tpmNonce := makeTpmNonce(instanceInfo, extraData) + + // Create an attestation using the loaded key + tpmAttestation, err := aK.Attest(tpmClient.AttestOpts{Nonce: tpmNonce[:]}) + if err != nil { + return nil, fmt.Errorf("creating attestation: %w", err) + } + attDoc := AttestationDocument{ Attestation: tpmAttestation, InstanceInfo: instanceInfo, @@ -147,7 +152,7 @@ func (i *Issuer) Issue(ctx context.Context, userData []byte, nonce []byte) (res return nil, fmt.Errorf("marshaling attestation document: %w", err) } - i.log.Infof("Successfully issued attestation statement") + i.log.Info("Successfully issued attestation statement") return rawAttDoc, nil } @@ -177,10 +182,10 @@ func NewValidator(expected measurements.M, getTrustedKey GetTPMTrustedAttestatio // Validate a TPM based attestation. func (v *Validator) Validate(ctx context.Context, attDocRaw []byte, nonce []byte) (userData []byte, err error) { - v.log.Infof("Validating attestation document") + v.log.Info("Validating attestation document") defer func() { if err != nil { - v.log.Warnf("Failed to validate attestation document: %s", err) + v.log.Warn(fmt.Sprintf("Failed to validate attestation document: %s", err)) } }() @@ -208,11 +213,13 @@ func (v *Validator) Validate(ctx context.Context, attDocRaw []byte, nonce []byte return nil, fmt.Errorf("validating attestation public key: %w", err) } + tpmNonce := makeTpmNonce(attDoc.InstanceInfo, extraData) + // Verify the TPM attestation state, err := tpmServer.VerifyAttestation( attDoc.Attestation, tpmServer.VerifyOpts{ - Nonce: extraData, + Nonce: tpmNonce[:], TrustedAKs: []crypto.PublicKey{aKP}, AllowSHA1: false, }, @@ -233,13 +240,13 @@ func (v *Validator) Validate(ctx context.Context, attDocRaw []byte, nonce []byte } warnings, errs := v.expected.Compare(attDoc.Attestation.Quotes[quoteIdx].Pcrs.Pcrs) for _, warning := range warnings { - v.log.Warnf(warning) + v.log.Warn(warning) } if len(errs) > 0 { return nil, fmt.Errorf("measurement validation failed:\n%w", errors.Join(errs...)) } - v.log.Infof("Successfully validated attestation document") + v.log.Info("Successfully validated attestation document") return attDoc.UserData, nil } @@ -287,3 +294,9 @@ func GetSelectedMeasurements(open TPMOpenFunc, selection tpm2.PCRSelection) (mea return m, nil } + +// makeTpmNonce creates a nonce for the TPM attestation and returns it in its marshaled form. +func makeTpmNonce(instanceInfo []byte, extraData []byte) [32]byte { + // Finding: GCP nonces cannot be larger than 32 bytes. + return sha256.Sum256(slices.Concat(instanceInfo, extraData)) +} diff --git a/internal/attestation/vtpm/attestation_test.go b/internal/attestation/vtpm/attestation_test.go index 9eeeef3b8..4b1e58b75 100644 --- a/internal/attestation/vtpm/attestation_test.go +++ b/internal/attestation/vtpm/attestation_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package vtpm @@ -90,7 +90,7 @@ func TestValidate(t *testing.T) { nonce := []byte{1, 2, 3, 4} challenge := []byte("Constellation") - ctx := context.Background() + ctx := t.Context() attDocRaw, err := issuer.Issue(ctx, challenge, nonce) require.NoError(err) @@ -308,7 +308,7 @@ func TestFailIssuer(t *testing.T) { "fail getAttestationKey": { issuer: NewIssuer( newSimTPMWithEventLog, - func(tpm io.ReadWriter) (*tpmclient.Key, error) { + func(_ io.ReadWriter) (*tpmclient.Key, error) { return nil, errors.New("failure") }, fakeGetInstanceInfo, @@ -320,7 +320,7 @@ func TestFailIssuer(t *testing.T) { "fail Attest": { issuer: NewIssuer( newSimTPMWithEventLog, - func(tpm io.ReadWriter) (*tpmclient.Key, error) { + func(_ io.ReadWriter) (*tpmclient.Key, error) { return &tpmclient.Key{}, nil }, fakeGetInstanceInfo, @@ -347,7 +347,7 @@ func TestFailIssuer(t *testing.T) { tc.issuer.log = logger.NewTest(t) - _, err := tc.issuer.Issue(context.Background(), tc.userData, tc.nonce) + _, err := tc.issuer.Issue(t.Context(), tc.userData, tc.nonce) assert.Error(err) }) } @@ -481,10 +481,10 @@ type testAttestationLogger struct { warnings []string } -func (w *testAttestationLogger) Infof(format string, args ...any) { +func (w *testAttestationLogger) Info(format string, args ...any) { w.infos = append(w.infos, fmt.Sprintf(format, args...)) } -func (w *testAttestationLogger) Warnf(format string, args ...any) { +func (w *testAttestationLogger) Warn(format string, args ...any) { w.warnings = append(w.warnings, fmt.Sprintf(format, args...)) } diff --git a/internal/attestation/vtpm/vtpm.go b/internal/attestation/vtpm/vtpm.go index 3a969eb2d..26b568043 100644 --- a/internal/attestation/vtpm/vtpm.go +++ b/internal/attestation/vtpm/vtpm.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/attestation/vtpm/vtpm_test.go b/internal/attestation/vtpm/vtpm_test.go index 18ded35ae..fece3a088 100644 --- a/internal/attestation/vtpm/vtpm_test.go +++ b/internal/attestation/vtpm/vtpm_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package vtpm @@ -13,5 +13,5 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } diff --git a/internal/cloud/aws/BUILD.bazel b/internal/cloud/aws/BUILD.bazel index b3ead651d..808e474ab 100644 --- a/internal/cloud/aws/BUILD.bazel +++ b/internal/cloud/aws/BUILD.bazel @@ -3,10 +3,7 @@ load("//bazel/go:go_test.bzl", "go_test") go_library( name = "aws", - srcs = [ - "aws.go", - "logger.go", - ], + srcs = ["aws.go"], importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/aws", visibility = ["//:__subpackages__"], deps = [ @@ -17,24 +14,18 @@ go_library( "@com_github_aws_aws_sdk_go_v2//aws", "@com_github_aws_aws_sdk_go_v2_config//:config", "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds", - "@com_github_aws_aws_sdk_go_v2_service_cloudwatchlogs//:cloudwatchlogs", - "@com_github_aws_aws_sdk_go_v2_service_cloudwatchlogs//types", "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2", "@com_github_aws_aws_sdk_go_v2_service_ec2//types", "@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//:elasticloadbalancingv2", "@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//types", "@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//:resourcegroupstaggingapi", "@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//types", - "@io_k8s_utils//clock", ], ) go_test( name = "aws_test", - srcs = [ - "aws_test.go", - "logger_test.go", - ], + srcs = ["aws_test.go"], embed = [":aws"], deps = [ "//internal/cloud", @@ -42,8 +33,6 @@ go_test( "//internal/role", "@com_github_aws_aws_sdk_go_v2//aws", "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds", - "@com_github_aws_aws_sdk_go_v2_service_cloudwatchlogs//:cloudwatchlogs", - "@com_github_aws_aws_sdk_go_v2_service_cloudwatchlogs//types", "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2", "@com_github_aws_aws_sdk_go_v2_service_ec2//types", "@com_github_aws_aws_sdk_go_v2_service_elasticloadbalancingv2//:elasticloadbalancingv2", @@ -51,8 +40,5 @@ go_test( "@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//:resourcegroupstaggingapi", "@com_github_aws_aws_sdk_go_v2_service_resourcegroupstaggingapi//types", "@com_github_stretchr_testify//assert", - "@com_github_stretchr_testify//require", - "@io_k8s_utils//clock/testing", - "@org_uber_go_goleak//:goleak", ], ) diff --git a/internal/cloud/aws/aws.go b/internal/cloud/aws/aws.go index f788aed03..3d0c8b316 100644 --- a/internal/cloud/aws/aws.go +++ b/internal/cloud/aws/aws.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -36,10 +36,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/role" ) -const ( - tagName = "Name" -) - type resourceAPI interface { GetResources(context.Context, *resourcegroupstaggingapi.GetResourcesInput, ...func(*resourcegroupstaggingapi.Options)) (*resourcegroupstaggingapi.GetResourcesOutput, error) } diff --git a/internal/cloud/aws/aws_test.go b/internal/cloud/aws/aws_test.go index e1b05ee88..d8b7541b7 100644 --- a/internal/cloud/aws/aws_test.go +++ b/internal/cloud/aws/aws_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package aws @@ -185,7 +185,7 @@ func TestSelf(t *testing.T) { ec2: tc.ec2API, } - self, err := m.Self(context.Background()) + self, err := m.Self(t.Context()) if tc.wantErr { assert.Error(err) return @@ -431,7 +431,7 @@ func TestList(t *testing.T) { ec2: tc.ec2, } - list, err := m.List(context.Background()) + list, err := m.List(t.Context()) if tc.wantErr { assert.Error(err) return @@ -694,7 +694,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) { ec2: successfulEC2, } - gotHost, gotPort, err := m.GetLoadBalancerEndpoint(context.Background()) + gotHost, gotPort, err := m.GetLoadBalancerEndpoint(t.Context()) if tc.wantErr { assert.Error(err) return diff --git a/internal/cloud/aws/logger.go b/internal/cloud/aws/logger.go deleted file mode 100644 index cbe0893de..000000000 --- a/internal/cloud/aws/logger.go +++ /dev/null @@ -1,228 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package aws - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" - logs "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/edgelesssys/constellation/v2/internal/cloud" - "k8s.io/utils/clock" -) - -// Logger is a Cloud Logger for AWS. -// Log messages are collected and periodically flushed to AWS Cloudwatch Logs. -type Logger struct { - api logAPI - - ec2API ec2API - imdsAPI imdsAPI - - groupName string - streamName string - - logs []types.InputLogEvent - sequenceToken *string - - mux sync.Mutex - interval time.Duration - clock clock.WithTicker - wg sync.WaitGroup - stopCh chan struct{} -} - -// NewLogger creates a new Cloud Logger for AWS. -func NewLogger(ctx context.Context) (*Logger, error) { - cfg, err := config.LoadDefaultConfig(ctx, config.WithEC2IMDSRegion()) - if err != nil { - return nil, err - } - client := logs.NewFromConfig(cfg) - - l := &Logger{ - api: client, - ec2API: ec2.NewFromConfig(cfg), - imdsAPI: imds.NewFromConfig(cfg), - interval: time.Second, - clock: clock.RealClock{}, - wg: sync.WaitGroup{}, - stopCh: make(chan struct{}, 1), - } - - if err := l.createStream(ctx); err != nil { - return nil, err - } - - l.flushLoop() - - return l, nil -} - -// Disclose adds a message to the log queue. -// The messages are flushed periodically to AWS Cloudwatch Logs. -func (l *Logger) Disclose(msg string) { - l.mux.Lock() - defer l.mux.Unlock() - l.logs = append(l.logs, types.InputLogEvent{ - Message: aws.String(msg), - Timestamp: aws.Int64(l.clock.Now().UnixMilli()), - }) -} - -// Close flushes the logs a final time and stops the flush loop. -func (l *Logger) Close() error { - l.stopCh <- struct{}{} - l.wg.Wait() - return l.flushLogs() -} - -// flushLogs flushes the aggregated log messages to AWS Cloudwatch Logs. -func (l *Logger) flushLogs() error { - // make sure only one flush operation is running at a time - l.mux.Lock() - defer l.mux.Unlock() - - if len(l.logs) == 0 { - return nil // no logs to flush - } - - ctx := context.Background() - logRequest := &logs.PutLogEventsInput{ - LogEvents: l.logs, - LogGroupName: &l.groupName, - LogStreamName: &l.streamName, - SequenceToken: l.sequenceToken, - } - - for res, err := l.api.PutLogEvents(ctx, logRequest); ; res, err = l.api.PutLogEvents(ctx, logRequest) { - if err == nil { - l.sequenceToken = res.NextSequenceToken - l.logs = nil - return nil - } - // If the flush operation was called on a pre-existing stream, - // or another operation sent logs to the same stream, - // the sequence token may not be set correctly. - // We can retrieve the correct sequence token from the error message. - var sequenceErr *types.InvalidSequenceTokenException - if !errors.As(err, &sequenceErr) { - return err - } - logRequest.SequenceToken = sequenceErr.ExpectedSequenceToken - } -} - -// flushLoop periodically flushes the logs to AWS Cloudwatch Logs. -func (l *Logger) flushLoop() { - l.wg.Add(1) - ticker := l.clock.NewTicker(l.interval) - - go func() { - defer l.wg.Done() - defer ticker.Stop() - - for { - _ = l.flushLogs() - select { - case <-ticker.C(): - case <-l.stopCh: - return - } - } - }() -} - -// createStream creates a new log stream in AWS Cloudwatch Logs. -func (l *Logger) createStream(ctx context.Context) error { - name, uid, err := l.getNameAndUID(ctx) - if err != nil { - return err - } - l.streamName = name - - // find log group with matching Constellation UID - describeInput := &logs.DescribeLogGroupsInput{} - for res, err := l.api.DescribeLogGroups(ctx, describeInput); ; res, err = l.api.DescribeLogGroups(ctx, describeInput) { - if err != nil { - return err - } - - for _, group := range res.LogGroups { - tags, err := l.api.ListTagsLogGroup(ctx, &logs.ListTagsLogGroupInput{LogGroupName: group.LogGroupName}) - if err != nil { - continue // we may not have permission to read the tags of a log group outside the Constellation scope - } - if tags.Tags[cloud.TagUID] == uid { - l.groupName = *group.LogGroupName - res.NextToken = nil // stop pagination - break - } - } - if res.NextToken == nil { - break - } - describeInput.NextToken = res.NextToken - } - if l.groupName == "" { - return fmt.Errorf("failed to find log group for UID %s", uid) - } - - // create or use existing log stream - if _, err := l.api.CreateLogStream(ctx, &logs.CreateLogStreamInput{ - LogGroupName: &l.groupName, - LogStreamName: &l.streamName, - }); err != nil { - // Ignore error if the stream already exists - var createErr *types.ResourceAlreadyExistsException - if !errors.As(err, &createErr) { - return err - } - } - - return nil -} - -func (l *Logger) getNameAndUID(ctx context.Context) (string, string, error) { - identity, err := l.imdsAPI.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{}) - if err != nil { - return "", "", fmt.Errorf("retrieving instance identity: %w", err) - } - - out, err := l.ec2API.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ - InstanceIds: []string{identity.InstanceID}, - }) - if err != nil { - return "", "", fmt.Errorf("descibing instances: %w", err) - } - - if len(out.Reservations) != 1 || len(out.Reservations[0].Instances) != 1 { - return "", "", fmt.Errorf("expected 1 instance, got %d", len(out.Reservations[0].Instances)) - } - - uid, err := findTag(out.Reservations[0].Instances[0].Tags, cloud.TagUID) - if err != nil { - return "", "", fmt.Errorf("finding tag %s: %w", cloud.TagUID, err) - } - - return identity.InstanceID, uid, err -} - -type logAPI interface { - CreateLogStream(context.Context, *logs.CreateLogStreamInput, ...func(*logs.Options)) (*logs.CreateLogStreamOutput, error) - DescribeLogGroups(context.Context, *logs.DescribeLogGroupsInput, ...func(*logs.Options)) (*logs.DescribeLogGroupsOutput, error) - ListTagsLogGroup(context.Context, *logs.ListTagsLogGroupInput, ...func(*logs.Options)) (*logs.ListTagsLogGroupOutput, error) - PutLogEvents(context.Context, *logs.PutLogEventsInput, ...func(*logs.Options)) (*logs.PutLogEventsOutput, error) -} diff --git a/internal/cloud/aws/logger_test.go b/internal/cloud/aws/logger_test.go deleted file mode 100644 index a5ca7d0a8..000000000 --- a/internal/cloud/aws/logger_test.go +++ /dev/null @@ -1,554 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package aws - -import ( - "context" - "errors" - "strconv" - "sync" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" - logs "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" - cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" - "github.com/aws/aws-sdk-go-v2/service/ec2" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/edgelesssys/constellation/v2/internal/cloud" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" - testclock "k8s.io/utils/clock/testing" -) - -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) -} - -func TestCreateStream(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - imdsAPI *stubIMDS - ec2API *stubEC2 - logs *stubLogs - wantGroup string - wantStream string - wantErr bool - }{ - "success new stream minimal": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - {LogGroupName: aws.String("test-group")}, - }, - }, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - }, - wantStream: "test-instance", - wantGroup: "test-group", - }, - "success one group of many": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - { - LogGroupName: aws.String("random-group"), - }, - { - LogGroupName: aws.String("other-group"), - }, - }, - NextToken: aws.String("next"), - }, - describeRes2: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - { - LogGroupName: aws.String("another-group"), - }, - { - LogGroupName: aws.String("test-group"), - }, - }, - }, - listTags: map[string]map[string]string{ - "random-group": { - "some-tag": "random-tag", - }, - "other-group": { - cloud.TagUID: "other-uid", - }, - "another-group": { - "some-tag": "uid", - }, - "test-group": { - cloud.TagUID: "uid", - }, - }, - }, - wantStream: "test-instance", - wantGroup: "test-group", - }, - "success stream exists": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - {LogGroupName: aws.String("test-group")}, - }, - }, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - createErr: &cloudwatchtypes.ResourceAlreadyExistsException{}, - }, - wantStream: "test-instance", - wantGroup: "test-group", - }, - "create stream error": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - {LogGroupName: aws.String("test-group")}, - }, - }, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - createErr: someErr, - }, - wantErr: true, - }, - "missing uid tag": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - {LogGroupName: aws.String("test-group")}, - }, - }, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - }, - wantErr: true, - }, - "missing identity document": { - imdsAPI: &stubIMDS{ - getInstanceIdentityDocumentErr: assert.AnError, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{ - LogGroups: []cloudwatchtypes.LogGroup{ - {LogGroupName: aws.String("test-group")}, - }, - }, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - }, - wantErr: true, - }, - "describe groups error": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeErr: someErr, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - }, - wantErr: true, - }, - "no matching groups": { - imdsAPI: &stubIMDS{ - instanceDocumentResp: &imds.GetInstanceIdentityDocumentOutput{ - InstanceIdentityDocument: imds.InstanceIdentityDocument{ - InstanceID: "test-instance", - }, - }, - }, - ec2API: &stubEC2{ - selfInstance: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: aws.String("test-instance"), - Tags: []ec2types.Tag{ - { - Key: aws.String(tagName), - Value: aws.String("test-instance"), - }, - { - Key: aws.String(cloud.TagUID), - Value: aws.String("uid"), - }, - }, - }, - }, - }, - }, - }, - }, - logs: &stubLogs{ - describeRes1: &logs.DescribeLogGroupsOutput{}, - listTags: map[string]map[string]string{"test-group": {cloud.TagUID: "uid"}}, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - l := &Logger{ - api: tc.logs, - imdsAPI: tc.imdsAPI, - ec2API: tc.ec2API, - } - - err := l.createStream(context.Background()) - if tc.wantErr { - assert.Error(err) - return - } - - assert.NoError(err) - assert.Equal(tc.wantGroup, l.groupName) - assert.Equal(tc.wantStream, l.streamName) - }) - } -} - -func TestLogging(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - logAPI := &stubLogs{} - - l := &Logger{ - api: logAPI, - interval: 1 * time.Millisecond, - clock: testclock.NewFakeClock(time.Time{}), - } - - l.Disclose("msg") - l.Disclose("msg") - // no logs until we flush to the API - assert.Len(logAPI.logs, 0) - - // flush - require.NoError(l.flushLogs()) - assert.Len(logAPI.logs, 2) - - // flushing doesn't do anything if there are no logs - require.NoError(l.flushLogs()) - assert.Len(logAPI.logs, 2) - - // if we flush with an incorrect sequence token, - // we should get a new sequence token and retry - logAPI.logSequenceToken = 15 - l.Disclose("msg") - require.NoError(l.flushLogs()) - assert.Len(logAPI.logs, 3) - - logAPI.putErr = errors.New("failed") - l.Disclose("msg") - assert.Error(l.flushLogs()) -} - -func TestFlushLoop(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - logAPI := &stubLogs{} - clock := testclock.NewFakeClock(time.Time{}) - - l := &Logger{ - api: logAPI, - interval: 1 * time.Second, - clock: clock, - stopCh: make(chan struct{}, 1), - } - - l.Disclose("msg") - l.Disclose("msg") - - l.flushLoop() - clock.Step(1 * time.Second) - require.NoError(l.Close()) - assert.Len(logAPI.logs, 2) -} - -func TestConcurrency(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - l := &Logger{ - api: &stubLogs{}, - interval: 1 * time.Second, - clock: testclock.NewFakeClock(time.Time{}), - stopCh: make(chan struct{}, 1), - } - var wg sync.WaitGroup - - wg.Add(100) - for i := 0; i < 100; i++ { - go func() { - defer wg.Done() - l.Disclose("msg") - }() - } - - wg.Wait() - assert.Len(l.logs, 100) - require.NoError(l.flushLogs()) - assert.Len(l.logs, 0) - - wg.Add(100) - for i := 0; i < 100; i++ { - go func() { - defer wg.Done() - l.Disclose("msg") - require.NoError(l.flushLogs()) - }() - } - wg.Wait() - assert.Len(l.logs, 0) -} - -type stubLogs struct { - createErr error - describeErr error - describeRes1 *logs.DescribeLogGroupsOutput - describeRes2 *logs.DescribeLogGroupsOutput - listTagsErr error - listTags map[string]map[string]string - putErr error - logSequenceToken int - logs []cloudwatchtypes.InputLogEvent -} - -func (s *stubLogs) CreateLogStream(context.Context, *logs.CreateLogStreamInput, ...func(*logs.Options)) (*logs.CreateLogStreamOutput, error) { - return nil, s.createErr -} - -func (s *stubLogs) DescribeLogGroups(_ context.Context, in *logs.DescribeLogGroupsInput, _ ...func(*logs.Options)) (*logs.DescribeLogGroupsOutput, error) { - if in.NextToken == nil { - return s.describeRes1, s.describeErr - } - return s.describeRes2, s.describeErr -} - -func (s *stubLogs) ListTagsLogGroup(_ context.Context, in *logs.ListTagsLogGroupInput, _ ...func(*logs.Options)) (*logs.ListTagsLogGroupOutput, error) { - return &logs.ListTagsLogGroupOutput{Tags: s.listTags[*in.LogGroupName]}, s.listTagsErr -} - -func (s *stubLogs) PutLogEvents(_ context.Context, in *logs.PutLogEventsInput, _ ...func(*logs.Options)) (*logs.PutLogEventsOutput, error) { - if s.putErr != nil { - return nil, s.putErr - } - if in.SequenceToken == nil || *in.SequenceToken == "" { - in.SequenceToken = aws.String("0") - } - gotSeq, err := strconv.Atoi(*in.SequenceToken) - if err != nil { - return nil, err - } - if gotSeq != s.logSequenceToken { - return nil, &cloudwatchtypes.InvalidSequenceTokenException{ExpectedSequenceToken: aws.String(strconv.Itoa(s.logSequenceToken))} - } - - s.logs = append(s.logs, in.LogEvents...) - s.logSequenceToken++ - - return &logs.PutLogEventsOutput{NextSequenceToken: aws.String(strconv.Itoa(s.logSequenceToken))}, nil -} diff --git a/internal/cloud/azure/BUILD.bazel b/internal/cloud/azure/BUILD.bazel index aad7461f1..29f4750d7 100644 --- a/internal/cloud/azure/BUILD.bazel +++ b/internal/cloud/azure/BUILD.bazel @@ -7,7 +7,8 @@ go_library( "azure.go", "imds.go", "interface.go", - "logger.go", + "iptables_cross.go", + "iptables_linux.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/azure", visibility = ["//:__subpackages__"], @@ -16,18 +17,20 @@ go_library( "//internal/cloud/azureshared", "//internal/cloud/metadata", "//internal/constants", - "//internal/logger", "//internal/role", "@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime", "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_applicationinsights_armapplicationinsights//:armapplicationinsights", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5//:armcompute", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4//:armnetwork", - "@com_github_microsoft_applicationinsights_go//appinsights", - "@io_k8s_kubernetes//pkg/util/iptables", - "@io_k8s_utils//exec", - "@org_uber_go_zap//:zap", - ], + "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v6//:armcompute", + "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v6//:armnetwork", + ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "@io_k8s_kubernetes//pkg/util/iptables", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "@io_k8s_kubernetes//pkg/util/iptables", + ], + "//conditions:default": [], + }), ) go_test( @@ -35,7 +38,6 @@ go_test( srcs = [ "azure_test.go", "imds_test.go", - "logger_test.go", ], embed = [":azure"], deps = [ @@ -44,10 +46,8 @@ go_test( "//internal/role", "@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime", "@com_github_azure_azure_sdk_for_go_sdk_azcore//to", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_applicationinsights_armapplicationinsights//:armapplicationinsights", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5//:armcompute", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v4//:armnetwork", - "@com_github_azure_go_autorest_autorest_to//:to", + "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v6//:armcompute", + "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_network_armnetwork_v6//:armnetwork", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//test/bufconn", diff --git a/internal/cloud/azure/azure.go b/internal/cloud/azure/azure.go index e979a9fe3..02b78c9b2 100644 --- a/internal/cloud/azure/azure.go +++ b/internal/cloud/azure/azure.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -23,17 +23,13 @@ import ( "strconv" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" "github.com/edgelesssys/constellation/v2/internal/cloud" "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/role" - "go.uber.org/zap" - "k8s.io/kubernetes/pkg/util/iptables" - "k8s.io/utils/exec" ) // Cloud provides Azure metadata and API access. @@ -396,8 +392,8 @@ func (c *Cloud) getLoadBalancerPublicIP(ctx context.Context) (string, error) { } /* -// TODO(malt3): uncomment and use as soon as we switch the primary endpoint to DNS. -// Addition from 3u13r: We have to think about how to handle DNS for internal load balancers +// TODO: uncomment and use as soon as we switch the primary endpoint to DNS. +// We have to think about how to handle DNS for internal load balancers // that only have a private IP address and therefore no DNS name by default. // // getLoadBalancerDNSName retrieves the dns name of the load balancer. @@ -440,68 +436,6 @@ func (c *Cloud) getLoadBalancerDNSName(ctx context.Context) (string, error) { } */ -// PrepareControlPlaneNode sets up iptables for the control plane node only -// if an internal load balancer is used. -// -// This is needed since during `kubeadm init` the API server must talk to the -// kubeAPIEndpoint, which is the load balancer IP address. During that time, the -// only healthy VM is the VM itself. Therefore, traffic is sent to the load balancer -// and the 5-tuple is (VM IP, , LB IP, 6443, TCP). -// Now the load balancer does not re-write the source IP address only the destination (DNAT). -// Therefore the 5-tuple is (VM IP, , VM IP, 6443, TCP). -// Now the VM responds to the SYN packet with a SYN-ACK packet, but the outgoing -// connection waits on a response from the load balancer and not the VM therefore -// dropping the packet. -// -// OpenShift also uses the same mechanism to redirect traffic to the API server: -// https://github.com/openshift/machine-config-operator/blob/e453bd20bac0e48afa74e9a27665abaf454d93cd/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml -func (c *Cloud) PrepareControlPlaneNode(ctx context.Context, log *logger.Logger) error { - selfMetadata, err := c.Self(ctx) - if err != nil { - return fmt.Errorf("failed to get self metadata: %w", err) - } - - // skipping iptables setup for worker nodes - if selfMetadata.Role != role.ControlPlane { - log.Infof("not a control plane node, skipping iptables setup") - return nil - } - - // skipping iptables setup if no internal LB exists e.g. - // for public LB architectures - loadbalancerIP, err := c.getLoadBalancerPrivateIP(ctx) - if err != nil { - log.With(zap.Error(err)).Warnf("skipping iptables setup, failed to get load balancer private IP") - return nil - } - - log.Infof("Setting up iptables for control plane node with load balancer IP %s", loadbalancerIP) - - iptablesExec := iptables.New(exec.New(), iptables.ProtocolIPv4) - if err != nil { - return fmt.Errorf("failed to create iptables client: %w", err) - } - - const chainName = "azure-lb-nat" - if _, err := iptablesExec.EnsureChain(iptables.TableNAT, chainName); err != nil { - return fmt.Errorf("failed to create iptables chain: %w", err) - } - - if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "PREROUTING", "-j", chainName); err != nil { - return fmt.Errorf("failed to add rule to iptables chain: %w", err) - } - - if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "OUTPUT", "-j", chainName); err != nil { - return fmt.Errorf("failed to add rule to iptables chain: %w", err) - } - - if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, chainName, "--dst", loadbalancerIP, "-p", "tcp", "--dport", "6443", "-j", "REDIRECT"); err != nil { - return fmt.Errorf("failed to add rule to iptables chain: %w", err) - } - - return nil -} - // convertToInstanceMetadata converts a armcomputev2.VirtualMachineScaleSetVM to a metadata.InstanceMetadata. func convertToInstanceMetadata(vm armcompute.VirtualMachineScaleSetVM, networkInterfaces []armnetwork.Interface, ) (metadata.InstanceMetadata, error) { diff --git a/internal/cloud/azure/azure_test.go b/internal/cloud/azure/azure_test.go index f5d0394c6..84182f9a0 100644 --- a/internal/cloud/azure/azure_test.go +++ b/internal/cloud/azure/azure_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azure @@ -13,8 +13,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" "github.com/edgelesssys/constellation/v2/internal/cloud" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/role" @@ -24,7 +24,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestGetInstance(t *testing.T) { @@ -150,7 +150,7 @@ func TestGetInstance(t *testing.T) { scaleSetsVMAPI: tc.scaleSetsVMAPI, netIfacAPI: tc.networkInterfacesAPI, } - instance, err := metadata.getInstance(context.Background(), tc.providerID) + instance, err := metadata.getInstance(t.Context(), tc.providerID) if tc.wantErr { assert.Error(err) return @@ -186,7 +186,7 @@ func TestUID(t *testing.T) { cloud := &Cloud{ imds: tc.imdsAPI, } - uid, err := cloud.UID(context.Background()) + uid, err := cloud.UID(t.Context()) if tc.wantErr { assert.Error(err) return @@ -222,7 +222,7 @@ func TestInitSecretHash(t *testing.T) { cloud := &Cloud{ imds: tc.imdsAPI, } - initSecretHash, err := cloud.InitSecretHash(context.Background()) + initSecretHash, err := cloud.InitSecretHash(t.Context()) if tc.wantErr { assert.Error(err) return @@ -410,7 +410,7 @@ func TestList(t *testing.T) { scaleSetsAPI: tc.scaleSetsAPI, scaleSetsVMAPI: tc.scaleSetsVMAPI, } - instances, err := azureMetadata.List(context.Background()) + instances, err := azureMetadata.List(t.Context()) if tc.wantErr { assert.Error(err) @@ -473,7 +473,7 @@ func TestGetNetworkSecurityGroupName(t *testing.T) { metadata := Cloud{ secGroupAPI: tc.securityGroupsAPI, } - name, err := metadata.getNetworkSecurityGroupName(context.Background(), "resource-group", "uid") + name, err := metadata.getNetworkSecurityGroupName(t.Context(), "resource-group", "uid") if tc.wantErr { assert.Error(err) return @@ -547,7 +547,7 @@ func TestGetSubnetworkCIDR(t *testing.T) { imds: tc.imdsAPI, virtNetAPI: tc.virtualNetworksAPI, } - subnetworkCIDR, err := metadata.getSubnetworkCIDR(context.Background()) + subnetworkCIDR, err := metadata.getSubnetworkCIDR(t.Context()) if tc.wantErr { assert.Error(err) return @@ -708,7 +708,7 @@ func TestGetLoadBalancerEndpoint(t *testing.T) { loadBalancerAPI: tc.loadBalancerAPI, pubIPAPI: tc.publicIPAddressesAPI, } - gotHost, gotPort, err := metadata.GetLoadBalancerEndpoint(context.Background()) + gotHost, gotPort, err := metadata.GetLoadBalancerEndpoint(t.Context()) if tc.wantErr { assert.Error(err) return diff --git a/internal/cloud/azure/imds.go b/internal/cloud/azure/imds.go index 21309b38a..6dea3d667 100644 --- a/internal/cloud/azure/imds.go +++ b/internal/cloud/azure/imds.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azure diff --git a/internal/cloud/azure/imds_test.go b/internal/cloud/azure/imds_test.go index b0afa517e..bf119f7b3 100644 --- a/internal/cloud/azure/imds_test.go +++ b/internal/cloud/azure/imds_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azure @@ -185,7 +185,7 @@ func TestIMDSClient(t *testing.T) { wantTags: defaultWantTags, }, "invalid imds response detected": { - server: newHTTPBufconnServer(func(writer http.ResponseWriter, request *http.Request) { + server: newHTTPBufconnServer(func(writer http.ResponseWriter, _ *http.Request) { fmt.Fprintln(writer, "invalid-result") }), wantProviderIDErr: true, @@ -214,7 +214,7 @@ func TestIMDSClient(t *testing.T) { } iClient := IMDSClient{client: &hClient} - ctx := context.Background() + ctx := t.Context() id, err := iClient.providerID(ctx) if tc.wantProviderIDErr { diff --git a/internal/cloud/azure/interface.go b/internal/cloud/azure/interface.go index 9f4b9f184..63a5afba9 100644 --- a/internal/cloud/azure/interface.go +++ b/internal/cloud/azure/interface.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azure @@ -10,9 +10,8 @@ import ( "context" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" ) type imdsAPI interface { @@ -74,9 +73,3 @@ type loadBalancerAPI interface { NewListPager(resourceGroupName string, options *armnetwork.LoadBalancersClientListOptions, ) *runtime.Pager[armnetwork.LoadBalancersClientListResponse] } - -type applicationInsightsAPI interface { - NewListByResourceGroupPager(resourceGroupName string, - options *armapplicationinsights.ComponentsClientListByResourceGroupOptions, - ) *runtime.Pager[armapplicationinsights.ComponentsClientListByResourceGroupResponse] -} diff --git a/internal/cloud/azure/iptables_cross.go b/internal/cloud/azure/iptables_cross.go new file mode 100644 index 000000000..d64b09db2 --- /dev/null +++ b/internal/cloud/azure/iptables_cross.go @@ -0,0 +1,19 @@ +//go:build !linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package azure + +import ( + "context" + "log/slog" +) + +// PrepareControlPlaneNode is only supported on Linux. +func (c *Cloud) PrepareControlPlaneNode(_ context.Context, _ *slog.Logger) error { + panic("azure.*Cloud.PrepareControlPlaneNode is only supported on Linux") +} diff --git a/internal/cloud/azure/iptables_linux.go b/internal/cloud/azure/iptables_linux.go new file mode 100644 index 000000000..c2d42a843 --- /dev/null +++ b/internal/cloud/azure/iptables_linux.go @@ -0,0 +1,76 @@ +//go:build linux + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package azure + +import ( + "context" + "fmt" + "log/slog" + + "github.com/edgelesssys/constellation/v2/internal/role" + "k8s.io/kubernetes/pkg/util/iptables" +) + +// PrepareControlPlaneNode sets up iptables for the control plane node only +// if an internal load balancer is used. +// +// This is needed since during `kubeadm init` the API server must talk to the +// kubeAPIEndpoint, which is the load balancer IP address. During that time, the +// only healthy VM is the VM itself. Therefore, traffic is sent to the load balancer +// and the 5-tuple is (VM IP, , LB IP, 6443, TCP). +// Now the load balancer does not re-write the source IP address only the destination (DNAT). +// Therefore the 5-tuple is (VM IP, , VM IP, 6443, TCP). +// Now the VM responds to the SYN packet with a SYN-ACK packet, but the outgoing +// connection waits on a response from the load balancer and not the VM therefore +// dropping the packet. +// +// OpenShift also uses the same mechanism to redirect traffic to the API server: +// https://github.com/openshift/machine-config-operator/blob/e453bd20bac0e48afa74e9a27665abaf454d93cd/templates/master/00-master/azure/files/opt-libexec-openshift-azure-routes-sh.yaml +func (c *Cloud) PrepareControlPlaneNode(ctx context.Context, log *slog.Logger) error { + selfMetadata, err := c.Self(ctx) + if err != nil { + return fmt.Errorf("failed to get self metadata: %w", err) + } + + // skipping iptables setup for worker nodes + if selfMetadata.Role != role.ControlPlane { + log.Info("not a control plane node, skipping iptables setup") + return nil + } + + // skipping iptables setup if no internal LB exists e.g. + // for public LB architectures + loadbalancerIP, err := c.getLoadBalancerPrivateIP(ctx) + if err != nil { + log.With(slog.Any("error", err)).Warn("skipping iptables setup, failed to get load balancer private IP") + return nil + } + + log.Info(fmt.Sprintf("Setting up iptables for control plane node with load balancer IP %s", loadbalancerIP)) + iptablesExec := iptables.New(iptables.ProtocolIPv4) + + const chainName = "azure-lb-nat" + if _, err := iptablesExec.EnsureChain(iptables.TableNAT, chainName); err != nil { + return fmt.Errorf("failed to create iptables chain: %w", err) + } + + if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "PREROUTING", "-j", chainName); err != nil { + return fmt.Errorf("failed to add rule to iptables chain: %w", err) + } + + if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, "OUTPUT", "-j", chainName); err != nil { + return fmt.Errorf("failed to add rule to iptables chain: %w", err) + } + + if _, err := iptablesExec.EnsureRule(iptables.Append, iptables.TableNAT, chainName, "--dst", loadbalancerIP, "-p", "tcp", "--dport", "6443", "-j", "REDIRECT"); err != nil { + return fmt.Errorf("failed to add rule to iptables chain: %w", err) + } + + return nil +} diff --git a/internal/cloud/azure/logger.go b/internal/cloud/azure/logger.go deleted file mode 100644 index 160a77b1d..000000000 --- a/internal/cloud/azure/logger.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "context" - "errors" - "fmt" - "net/http" - - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights" - "github.com/edgelesssys/constellation/v2/internal/cloud" - "github.com/microsoft/ApplicationInsights-Go/appinsights" -) - -// Logger implements CloudLogger interface for Azure to Disclose early boot -// logs into Azure's App Insights service. -type Logger struct { - client appinsights.TelemetryClient -} - -// NewLogger creates a new client to store information in Azure Application Insights -// https://github.com/Microsoft/ApplicationInsights-go -func NewLogger(ctx context.Context) (*Logger, error) { - cred, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - return nil, fmt.Errorf("loading credentials: %w", err) - } - imdsAPI := &IMDSClient{ - client: &http.Client{Transport: &http.Transport{Proxy: nil}}, - } - subscriptionID, err := imdsAPI.subscriptionID(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving subscription ID: %w", err) - } - appInsightAPI, err := armapplicationinsights.NewComponentsClient(subscriptionID, cred, nil) - if err != nil { - return nil, fmt.Errorf("setting up insights API client. %w", err) - } - - instrumentationKey, err := getAppInsightsKey(ctx, imdsAPI, appInsightAPI) - if err != nil { - return nil, fmt.Errorf("getting app insights instrumentation key: %w", err) - } - - client := appinsights.NewTelemetryClient(instrumentationKey) - - name, err := imdsAPI.name(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving instance name: %w", err) - } - client.Context().CommonProperties["instance-name"] = name - - return &Logger{client: client}, nil -} - -// Disclose stores log information in Azure Application Insights! -// Do **NOT** log sensitive information! -func (l *Logger) Disclose(msg string) { - l.client.Track(appinsights.NewTraceTelemetry(msg, appinsights.Information)) -} - -// Close blocks until all information are written to cloud API. -func (l *Logger) Close() error { - <-l.client.Channel().Close() - return nil -} - -// getAppInsightsKey returns a instrumentation key needed to set up cloud logging on Azure. -// The key is retrieved from the resource group of the instance the function is called from. -func getAppInsightsKey(ctx context.Context, imdsAPI imdsAPI, appInsightAPI applicationInsightsAPI) (string, error) { - resourceGroup, err := imdsAPI.resourceGroup(ctx) - if err != nil { - return "", err - } - uid, err := imdsAPI.uid(ctx) - if err != nil { - return "", err - } - - pager := appInsightAPI.NewListByResourceGroupPager(resourceGroup, nil) - for pager.More() { - page, err := pager.NextPage(ctx) - if err != nil { - return "", fmt.Errorf("retrieving application insights: %w", err) - } - - for _, component := range page.Value { - if component == nil || component.Tags == nil || - component.Tags[cloud.TagUID] == nil || *component.Tags[cloud.TagUID] != uid { - continue - } - - if component.Properties == nil || component.Properties.InstrumentationKey == nil { - return "", errors.New("unable to get instrumentation key") - } - return *component.Properties.InstrumentationKey, nil - } - } - return "", errors.New("could not find correctly tagged application insights") -} diff --git a/internal/cloud/azure/logger_test.go b/internal/cloud/azure/logger_test.go deleted file mode 100644 index a033c9c47..000000000 --- a/internal/cloud/azure/logger_test.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "context" - "errors" - "testing" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/applicationinsights/armapplicationinsights" - "github.com/Azure/go-autorest/autorest/to" - "github.com/edgelesssys/constellation/v2/internal/cloud" - "github.com/stretchr/testify/assert" -) - -func TestGetAppInsightsKey(t *testing.T) { - someErr := errors.New("failed") - goodAppInsights := armapplicationinsights.Component{ - Tags: map[string]*string{ - cloud.TagUID: to.StringPtr("uid"), - }, - Properties: &armapplicationinsights.ComponentProperties{ - InstrumentationKey: to.StringPtr("key"), - }, - } - - testCases := map[string]struct { - imds *stubIMDSAPI - appInsights *stubApplicationsInsightsAPI - wantKey string - wantErr bool - }{ - "success": { - imds: &stubIMDSAPI{ - resourceGroupVal: "resource-group", - uidVal: "uid", - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{list: []armapplicationinsights.Component{goodAppInsights}}, - }, - wantKey: "key", - }, - "multiple apps": { - imds: &stubIMDSAPI{ - resourceGroupVal: "resource-group", - uidVal: "uid", - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{list: []armapplicationinsights.Component{ - { - Tags: map[string]*string{ - cloud.TagUID: to.StringPtr("different-uid"), - }, - Properties: &armapplicationinsights.ComponentProperties{ - InstrumentationKey: to.StringPtr("different-key"), - }, - }, - goodAppInsights, - }}, - }, - wantKey: "key", - }, - "missing properties": { - imds: &stubIMDSAPI{ - resourceGroupVal: "resource-group", - uidVal: "uid", - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{list: []armapplicationinsights.Component{ - { - Tags: map[string]*string{ - cloud.TagUID: to.StringPtr("uid"), - }, - }, - }}, - }, - wantErr: true, - }, - "no app with matching uid": { - imds: &stubIMDSAPI{ - resourceGroupVal: "resource-group", - uidVal: "uid", - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{list: []armapplicationinsights.Component{ - { - Tags: map[string]*string{ - cloud.TagUID: to.StringPtr("different-uid"), - }, - Properties: &armapplicationinsights.ComponentProperties{ - InstrumentationKey: to.StringPtr("different-key"), - }, - }, - }}, - }, - wantErr: true, - }, - "imds resource group error": { - imds: &stubIMDSAPI{ - resourceGroupErr: someErr, - uidVal: "uid", - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{list: []armapplicationinsights.Component{goodAppInsights}}, - }, - wantErr: true, - }, - "imds uid error": { - imds: &stubIMDSAPI{ - resourceGroupVal: "resource-group", - uidErr: someErr, - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{list: []armapplicationinsights.Component{goodAppInsights}}, - }, - wantErr: true, - }, - "app insights list error": { - imds: &stubIMDSAPI{ - resourceGroupVal: "resource-group", - uidVal: "uid", - }, - appInsights: &stubApplicationsInsightsAPI{ - pager: &stubApplicationKeyPager{fetchErr: someErr}, - }, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - key, err := getAppInsightsKey(context.Background(), tc.imds, tc.appInsights) - if tc.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tc.wantKey, key) - } - }) - } -} - -type stubApplicationKeyPager struct { - list []armapplicationinsights.Component - fetchErr error - more bool -} - -func (p *stubApplicationKeyPager) moreFunc() func(armapplicationinsights.ComponentsClientListByResourceGroupResponse) bool { - return func(armapplicationinsights.ComponentsClientListByResourceGroupResponse) bool { - return p.more - } -} - -func (p *stubApplicationKeyPager) fetcherFunc() func(context.Context, *armapplicationinsights.ComponentsClientListByResourceGroupResponse, -) (armapplicationinsights.ComponentsClientListByResourceGroupResponse, error) { - return func(context.Context, *armapplicationinsights.ComponentsClientListByResourceGroupResponse) (armapplicationinsights.ComponentsClientListByResourceGroupResponse, error) { - page := make([]*armapplicationinsights.Component, len(p.list)) - for i := range p.list { - page[i] = &p.list[i] - } - return armapplicationinsights.ComponentsClientListByResourceGroupResponse{ - ComponentListResult: armapplicationinsights.ComponentListResult{ - Value: page, - }, - }, p.fetchErr - } -} - -type stubApplicationsInsightsAPI struct { - pager *stubApplicationKeyPager -} - -func (a *stubApplicationsInsightsAPI) NewListByResourceGroupPager(_ string, _ *armapplicationinsights.ComponentsClientListByResourceGroupOptions, -) *runtime.Pager[armapplicationinsights.ComponentsClientListByResourceGroupResponse] { - return runtime.NewPager(runtime.PagingHandler[armapplicationinsights.ComponentsClientListByResourceGroupResponse]{ - More: a.pager.moreFunc(), - Fetcher: a.pager.fetcherFunc(), - }) -} diff --git a/internal/cloud/azureshared/appcredentials.go b/internal/cloud/azureshared/appcredentials.go index 7c6c7ec65..fe5c8b6d9 100644 --- a/internal/cloud/azureshared/appcredentials.go +++ b/internal/cloud/azureshared/appcredentials.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azureshared diff --git a/internal/cloud/azureshared/appcredentials_test.go b/internal/cloud/azureshared/appcredentials_test.go index 086a16d8f..27cddf96f 100644 --- a/internal/cloud/azureshared/appcredentials_test.go +++ b/internal/cloud/azureshared/appcredentials_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azureshared @@ -16,7 +16,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestApplicationCredentialsFromURI(t *testing.T) { diff --git a/internal/cloud/azureshared/azureshared.go b/internal/cloud/azureshared/azureshared.go index 38cf82b27..f8909e3b7 100644 --- a/internal/cloud/azureshared/azureshared.go +++ b/internal/cloud/azureshared/azureshared.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/cloud/azureshared/metadata.go b/internal/cloud/azureshared/metadata.go index b21ffd4c3..1289adc8d 100644 --- a/internal/cloud/azureshared/metadata.go +++ b/internal/cloud/azureshared/metadata.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azureshared diff --git a/internal/cloud/azureshared/metadata_test.go b/internal/cloud/azureshared/metadata_test.go index 61d71a94d..a7c562fe6 100644 --- a/internal/cloud/azureshared/metadata_test.go +++ b/internal/cloud/azureshared/metadata_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azureshared diff --git a/internal/cloud/cloud.go b/internal/cloud/cloud.go index e7e9a88ca..7d255bd19 100644 --- a/internal/cloud/cloud.go +++ b/internal/cloud/cloud.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/cloud/cloudprovider/cloudprovider.go b/internal/cloud/cloudprovider/cloudprovider.go index 47791f943..c25b18d69 100644 --- a/internal/cloud/cloudprovider/cloudprovider.go +++ b/internal/cloud/cloudprovider/cloudprovider.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudprovider @@ -16,6 +16,9 @@ import ( // Provider is cloud provider used by the CLI. type Provider uint32 +// Tags is the type that holds additional tags for cloud resources. +type Tags map[string]string + const ( // Unknown is default value for Provider. Unknown Provider = iota diff --git a/internal/cloud/cloudprovider/cloudprovider_test.go b/internal/cloud/cloudprovider/cloudprovider_test.go index b8c7a4d40..e7154195c 100644 --- a/internal/cloud/cloudprovider/cloudprovider_test.go +++ b/internal/cloud/cloudprovider/cloudprovider_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cloudprovider diff --git a/internal/cloud/gcp/BUILD.bazel b/internal/cloud/gcp/BUILD.bazel index fda87cca9..138f0c625 100644 --- a/internal/cloud/gcp/BUILD.bazel +++ b/internal/cloud/gcp/BUILD.bazel @@ -6,7 +6,6 @@ go_library( srcs = [ "gcp.go", "interface.go", - "logger.go", "wrappers.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/gcp", @@ -20,7 +19,6 @@ go_library( "@com_google_cloud_go_compute//apiv1", "@com_google_cloud_go_compute//apiv1/computepb", "@com_google_cloud_go_compute_metadata//:metadata", - "@com_google_cloud_go_logging//:logging", "@org_golang_google_api//iterator", "@org_golang_google_protobuf//proto", ], diff --git a/internal/cloud/gcp/gcp.go b/internal/cloud/gcp/gcp.go index ba689ef3a..f38d848f6 100644 --- a/internal/cloud/gcp/gcp.go +++ b/internal/cloud/gcp/gcp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/cloud/gcp/gcp_test.go b/internal/cloud/gcp/gcp_test.go index 7ac614065..4066f3c5b 100644 --- a/internal/cloud/gcp/gcp_test.go +++ b/internal/cloud/gcp/gcp_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcp @@ -27,6 +27,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -171,7 +172,7 @@ func TestGetInstance(t *testing.T) { instanceAPI: &tc.instanceAPI, subnetAPI: &tc.subnetAPI, } - instance, err := cloud.getInstance(context.Background(), tc.projectID, tc.zone, tc.instanceName) + instance, err := cloud.getInstance(t.Context(), tc.projectID, tc.zone, tc.instanceName) if tc.wantErr { assert.Error(err) @@ -473,7 +474,7 @@ func TestGetLoadbalancerEndpoint(t *testing.T) { regionalForwardingRulesAPI: &tc.regionalForwardingRulesAPI, } - gotHost, gotPort, err := cloud.GetLoadBalancerEndpoint(context.Background()) + gotHost, gotPort, err := cloud.GetLoadBalancerEndpoint(t.Context()) if tc.wantErr { assert.Error(err) return @@ -809,7 +810,7 @@ func TestList(t *testing.T) { zoneAPI: &tc.zoneAPI, } - instances, err := cloud.List(context.Background()) + instances, err := cloud.List(t.Context()) if tc.wantErr { assert.Error(err) return @@ -914,7 +915,7 @@ func TestZones(t *testing.T) { assert.Empty(cloud.zoneCache) - gotZones, err := cloud.zones(context.Background(), "someProject", "someregion-west3") + gotZones, err := cloud.zones(t.Context(), "someProject", "someregion-west3") if tc.wantErr { assert.Error(err) return @@ -1065,7 +1066,7 @@ func TestUID(t *testing.T) { instanceAPI: &tc.instanceAPI, } - uid, err := cloud.UID(context.Background()) + uid, err := cloud.UID(t.Context()) if tc.wantErr { assert.Error(err) return @@ -1169,7 +1170,7 @@ func TestInitSecretHash(t *testing.T) { instanceAPI: &tc.instanceAPI, } - initSecretHash, err := cloud.InitSecretHash(context.Background()) + initSecretHash, err := cloud.InitSecretHash(t.Context()) if tc.wantErr { assert.Error(err) return diff --git a/internal/cloud/gcp/interface.go b/internal/cloud/gcp/interface.go index e78c9861b..ad9f131f7 100644 --- a/internal/cloud/gcp/interface.go +++ b/internal/cloud/gcp/interface.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcp diff --git a/internal/cloud/gcp/logger.go b/internal/cloud/gcp/logger.go deleted file mode 100644 index c366c63dd..000000000 --- a/internal/cloud/gcp/logger.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "context" - "fmt" - "log" - - "cloud.google.com/go/compute/metadata" - "cloud.google.com/go/logging" -) - -// Logger logs to GCP cloud logging. Do not use to log sensitive information. -type Logger struct { - client *logging.Client - logger *log.Logger -} - -// NewLogger creates a new Cloud Logger for GCP. -// https://cloud.google.com/logging/docs/setup/go -func NewLogger(ctx context.Context, logName string) (*Logger, error) { - projectID, err := metadata.NewClient(nil).ProjectID() - if err != nil { - return nil, fmt.Errorf("retrieving project ID from imds: %w", err) - } - - client, err := logging.NewClient(ctx, projectID) - if err != nil { - return nil, err - } - - logger := client.Logger(logName).StandardLogger(logging.Info) - - return &Logger{ - client: client, - logger: logger, - }, nil -} - -// Disclose stores log information in GCP Cloud Logging! Do **NOT** log sensitive -// information! -func (l *Logger) Disclose(msg string) { - l.logger.Println(msg) -} - -// Close waits for all buffer to be written. -func (l *Logger) Close() error { - return l.client.Close() -} diff --git a/internal/cloud/gcp/wrappers.go b/internal/cloud/gcp/wrappers.go index 72b46399a..b8d60b92a 100644 --- a/internal/cloud/gcp/wrappers.go +++ b/internal/cloud/gcp/wrappers.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcp diff --git a/internal/cloud/gcpshared/gcpshared.go b/internal/cloud/gcpshared/gcpshared.go index 93b4a41fa..667ad4647 100644 --- a/internal/cloud/gcpshared/gcpshared.go +++ b/internal/cloud/gcpshared/gcpshared.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/cloud/gcpshared/providerid.go b/internal/cloud/gcpshared/providerid.go index 70c26334c..0a32af728 100644 --- a/internal/cloud/gcpshared/providerid.go +++ b/internal/cloud/gcpshared/providerid.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcpshared diff --git a/internal/cloud/gcpshared/providerid_test.go b/internal/cloud/gcpshared/providerid_test.go index 61fbdb2f5..1ebeb8e86 100644 --- a/internal/cloud/gcpshared/providerid_test.go +++ b/internal/cloud/gcpshared/providerid_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcpshared diff --git a/internal/cloud/gcpshared/serviceaccountkey.go b/internal/cloud/gcpshared/serviceaccountkey.go index bb90cdde2..6908b385d 100644 --- a/internal/cloud/gcpshared/serviceaccountkey.go +++ b/internal/cloud/gcpshared/serviceaccountkey.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcpshared diff --git a/internal/cloud/gcpshared/serviceaccountkey_test.go b/internal/cloud/gcpshared/serviceaccountkey_test.go index 54cf7296b..de77f6ff2 100644 --- a/internal/cloud/gcpshared/serviceaccountkey_test.go +++ b/internal/cloud/gcpshared/serviceaccountkey_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcpshared diff --git a/internal/cloud/metadata/metadata.go b/internal/cloud/metadata/metadata.go index 7b3aed893..fe2b844c5 100644 --- a/internal/cloud/metadata/metadata.go +++ b/internal/cloud/metadata/metadata.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package metadata diff --git a/internal/cloud/openstack/BUILD.bazel b/internal/cloud/openstack/BUILD.bazel index e1cc8dbcd..719f3fe02 100644 --- a/internal/cloud/openstack/BUILD.bazel +++ b/internal/cloud/openstack/BUILD.bazel @@ -18,11 +18,12 @@ go_library( "//internal/cloud/metadata", "//internal/constants", "//internal/role", - "@com_github_gophercloud_gophercloud//:gophercloud", - "@com_github_gophercloud_gophercloud//openstack/compute/v2/servers", - "@com_github_gophercloud_gophercloud//openstack/networking/v2/subnets", - "@com_github_gophercloud_gophercloud//pagination", - "@com_github_gophercloud_utils//openstack/clientconfig", + "@com_github_gophercloud_gophercloud_v2//:gophercloud", + "@com_github_gophercloud_gophercloud_v2//openstack/compute/v2/servers", + "@com_github_gophercloud_gophercloud_v2//openstack/networking/v2/networks", + "@com_github_gophercloud_gophercloud_v2//openstack/networking/v2/subnets", + "@com_github_gophercloud_gophercloud_v2//pagination", + "@com_github_gophercloud_utils_v2//openstack/clientconfig", ], ) @@ -39,10 +40,11 @@ go_test( deps = [ "//internal/cloud/metadata", "//internal/role", - "@com_github_gophercloud_gophercloud//:gophercloud", - "@com_github_gophercloud_gophercloud//openstack/compute/v2/servers", - "@com_github_gophercloud_gophercloud//openstack/networking/v2/subnets", - "@com_github_gophercloud_gophercloud//pagination", + "@com_github_gophercloud_gophercloud_v2//:gophercloud", + "@com_github_gophercloud_gophercloud_v2//openstack/compute/v2/servers", + "@com_github_gophercloud_gophercloud_v2//openstack/networking/v2/networks", + "@com_github_gophercloud_gophercloud_v2//openstack/networking/v2/subnets", + "@com_github_gophercloud_gophercloud_v2//pagination", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", ], diff --git a/internal/cloud/openstack/accountkey.go b/internal/cloud/openstack/accountkey.go index d781091e9..49d359e6f 100644 --- a/internal/cloud/openstack/accountkey.go +++ b/internal/cloud/openstack/accountkey.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack diff --git a/internal/cloud/openstack/accountkey_test.go b/internal/cloud/openstack/accountkey_test.go index e9805e1d9..b15da3a81 100644 --- a/internal/cloud/openstack/accountkey_test.go +++ b/internal/cloud/openstack/accountkey_test.go @@ -1,6 +1,6 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack diff --git a/internal/cloud/openstack/api.go b/internal/cloud/openstack/api.go index e66c31498..839a47fbb 100644 --- a/internal/cloud/openstack/api.go +++ b/internal/cloud/openstack/api.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack @@ -10,9 +10,10 @@ import ( "context" "github.com/edgelesssys/constellation/v2/internal/role" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - "github.com/gophercloud/gophercloud/pagination" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" + "github.com/gophercloud/gophercloud/v2/pagination" ) type imdsAPI interface { @@ -23,14 +24,15 @@ type imdsAPI interface { initSecretHash(ctx context.Context) (string, error) role(ctx context.Context) (role.Role, error) vpcIP(ctx context.Context) (string, error) - networkIDs(ctx context.Context) ([]string, error) + loadBalancerEndpoint(ctx context.Context) (string, error) } type serversAPI interface { ListServers(opts servers.ListOptsBuilder) pagerAPI + ListNetworks(opts networks.ListOptsBuilder) pagerAPI ListSubnets(opts subnets.ListOpts) pagerAPI } type pagerAPI interface { - AllPages() (pagination.Page, error) + AllPages(context.Context) (pagination.Page, error) } diff --git a/internal/cloud/openstack/api_test.go b/internal/cloud/openstack/api_test.go index b6f623296..9acb07494 100644 --- a/internal/cloud/openstack/api_test.go +++ b/internal/cloud/openstack/api_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack @@ -10,28 +10,29 @@ import ( "context" "github.com/edgelesssys/constellation/v2/internal/role" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - "github.com/gophercloud/gophercloud/pagination" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" + "github.com/gophercloud/gophercloud/v2/pagination" ) type stubIMDSClient struct { - providerIDResult string - providerIDErr error - nameResult string - nameErr error - projectIDResult string - projectIDErr error - uidResult string - uidErr error - initSecretHashResult string - initSecretHashErr error - roleResult role.Role - roleErr error - vpcIPResult string - vpcIPErr error - networkIDsResult []string - networkIDsErr error + providerIDResult string + providerIDErr error + nameResult string + nameErr error + projectIDResult string + projectIDErr error + uidResult string + uidErr error + initSecretHashResult string + initSecretHashErr error + roleResult role.Role + roleErr error + vpcIPResult string + vpcIPErr error + loadBalancerEndpointResult string + loadBalancerEndpointErr error } func (c *stubIMDSClient) providerID(_ context.Context) (string, error) { @@ -62,12 +63,13 @@ func (c *stubIMDSClient) vpcIP(_ context.Context) (string, error) { return c.vpcIPResult, c.vpcIPErr } -func (c *stubIMDSClient) networkIDs(_ context.Context) ([]string, error) { - return c.networkIDsResult, c.networkIDsErr +func (c *stubIMDSClient) loadBalancerEndpoint(_ context.Context) (string, error) { + return c.loadBalancerEndpointResult, c.loadBalancerEndpointErr } type stubServersClient struct { serversPager stubPager + netsPager stubPager subnetsPager stubPager } @@ -75,6 +77,10 @@ func (c *stubServersClient) ListServers(_ servers.ListOptsBuilder) pagerAPI { return &c.serversPager } +func (c *stubServersClient) ListNetworks(_ networks.ListOptsBuilder) pagerAPI { + return &c.netsPager +} + func (c *stubServersClient) ListSubnets(_ subnets.ListOpts) pagerAPI { return &c.subnetsPager } @@ -84,6 +90,6 @@ type stubPager struct { allPagesErr error } -func (p *stubPager) AllPages() (pagination.Page, error) { +func (p *stubPager) AllPages(_ context.Context) (pagination.Page, error) { return p.page, p.allPagesErr } diff --git a/internal/cloud/openstack/clouds/BUILD.bazel b/internal/cloud/openstack/clouds/BUILD.bazel new file mode 100644 index 000000000..153bed763 --- /dev/null +++ b/internal/cloud/openstack/clouds/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "clouds", + srcs = [ + "clouds.go", + "read.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/openstack/clouds", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/file", + "@com_github_mitchellh_go_homedir//:go-homedir", + ], +) diff --git a/internal/cloud/openstack/clouds/LICENSE b/internal/cloud/openstack/clouds/LICENSE new file mode 100644 index 000000000..b1da7201f --- /dev/null +++ b/internal/cloud/openstack/clouds/LICENSE @@ -0,0 +1,193 @@ +Copyright 2012-2013 Rackspace, Inc. +Copyright Gophercloud authors +Copyright (c) Edgeless Systems GmbH + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/internal/cloud/openstack/clouds/clouds.go b/internal/cloud/openstack/clouds/clouds.go new file mode 100644 index 000000000..923325fe2 --- /dev/null +++ b/internal/cloud/openstack/clouds/clouds.go @@ -0,0 +1,208 @@ +/* +Copyright 2012-2013 Rackspace, Inc. +Copyright Gophercloud authors +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: Apache-2.0 +*/ +package clouds + +import "encoding/json" + +// Clouds represents a collection of Cloud entries in a clouds.yaml file. +type Clouds struct { + Clouds map[string]Cloud `yaml:"clouds" json:"clouds"` +} + +// Cloud represents an entry in a clouds.yaml/public-clouds.yaml/secure.yaml file. +type Cloud struct { + Cloud string `yaml:"cloud,omitempty" json:"cloud,omitempty"` + Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` + AuthInfo *AuthInfo `yaml:"auth,omitempty" json:"auth,omitempty"` + AuthType AuthType `yaml:"auth_type,omitempty" json:"auth_type,omitempty"` + RegionName string `yaml:"region_name,omitempty" json:"region_name,omitempty"` + Regions []Region `yaml:"regions,omitempty" json:"regions,omitempty"` + + // EndpointType and Interface both specify whether to use the public, internal, + // or admin interface of a service. They should be considered synonymous, but + // EndpointType will take precedence when both are specified. + EndpointType string `yaml:"endpoint_type,omitempty" json:"endpoint_type,omitempty"` + Interface string `yaml:"interface,omitempty" json:"interface,omitempty"` + + // API Version overrides. + IdentityAPIVersion string `yaml:"identity_api_version,omitempty" json:"identity_api_version,omitempty"` + VolumeAPIVersion string `yaml:"volume_api_version,omitempty" json:"volume_api_version,omitempty"` + + // Verify whether or not SSL API requests should be verified. + Verify *bool `yaml:"verify,omitempty" json:"verify,omitempty"` + + // CACertFile a path to a CA Cert bundle that can be used as part of + // verifying SSL API requests. + CACertFile string `yaml:"cacert,omitempty" json:"cacert,omitempty"` + + // ClientCertFile a path to a client certificate to use as part of the SSL + // transaction. + ClientCertFile string `yaml:"cert,omitempty" json:"cert,omitempty"` + + // ClientKeyFile a path to a client key to use as part of the SSL + // transaction. + ClientKeyFile string `yaml:"key,omitempty" json:"key,omitempty"` +} + +// AuthInfo represents the auth section of a cloud entry or +// auth options entered explicitly in ClientOpts. +type AuthInfo struct { + // AuthURL is the keystone/identity endpoint URL. + AuthURL string `yaml:"auth_url,omitempty" json:"auth_url,omitempty"` + + // Token is a pre-generated authentication token. + Token string `yaml:"token,omitempty" json:"token,omitempty"` + + // Username is the username of the user. + Username string `yaml:"username,omitempty" json:"username,omitempty"` + + // UserID is the unique ID of a user. + UserID string `yaml:"user_id,omitempty" json:"user_id,omitempty"` + + // Password is the password of the user. + Password string `yaml:"password,omitempty" json:"password,omitempty"` + + // Application Credential ID to login with. + ApplicationCredentialID string `yaml:"application_credential_id,omitempty" json:"application_credential_id,omitempty"` + + // Application Credential name to login with. + ApplicationCredentialName string `yaml:"application_credential_name,omitempty" json:"application_credential_name,omitempty"` + + // Application Credential secret to login with. + ApplicationCredentialSecret string `yaml:"application_credential_secret,omitempty" json:"application_credential_secret,omitempty"` + + // SystemScope is a system information to scope to. + SystemScope string `yaml:"system_scope,omitempty" json:"system_scope,omitempty"` + + // ProjectName is the common/human-readable name of a project. + // Users can be scoped to a project. + // ProjectName on its own is not enough to ensure a unique scope. It must + // also be combined with either a ProjectDomainName or ProjectDomainID. + // ProjectName cannot be combined with ProjectID in a scope. + ProjectName string `yaml:"project_name,omitempty" json:"project_name,omitempty"` + + // ProjectID is the unique ID of a project. + // It can be used to scope a user to a specific project. + ProjectID string `yaml:"project_id,omitempty" json:"project_id,omitempty"` + + // UserDomainName is the name of the domain where a user resides. + // It is used to identify the source domain of a user. + UserDomainName string `yaml:"user_domain_name,omitempty" json:"user_domain_name,omitempty"` + + // UserDomainID is the unique ID of the domain where a user resides. + // It is used to identify the source domain of a user. + UserDomainID string `yaml:"user_domain_id,omitempty" json:"user_domain_id,omitempty"` + + // ProjectDomainName is the name of the domain where a project resides. + // It is used to identify the source domain of a project. + // ProjectDomainName can be used in addition to a ProjectName when scoping + // a user to a specific project. + ProjectDomainName string `yaml:"project_domain_name,omitempty" json:"project_domain_name,omitempty"` + + // ProjectDomainID is the name of the domain where a project resides. + // It is used to identify the source domain of a project. + // ProjectDomainID can be used in addition to a ProjectName when scoping + // a user to a specific project. + ProjectDomainID string `yaml:"project_domain_id,omitempty" json:"project_domain_id,omitempty"` + + // DomainName is the name of a domain which can be used to identify the + // source domain of either a user or a project. + // If UserDomainName and ProjectDomainName are not specified, then DomainName + // is used as a default choice. + // It can also be used be used to specify a domain-only scope. + DomainName string `yaml:"domain_name,omitempty" json:"domain_name,omitempty"` + + // DomainID is the unique ID of a domain which can be used to identify the + // source domain of eitehr a user or a project. + // If UserDomainID and ProjectDomainID are not specified, then DomainID is + // used as a default choice. + // It can also be used be used to specify a domain-only scope. + DomainID string `yaml:"domain_id,omitempty" json:"domain_id,omitempty"` + + // DefaultDomain is the domain ID to fall back on if no other domain has + // been specified and a domain is required for scope. + DefaultDomain string `yaml:"default_domain,omitempty" json:"default_domain,omitempty"` + + // AllowReauth should be set to true if you grant permission for Gophercloud to + // cache your credentials in memory, and to allow Gophercloud to attempt to + // re-authenticate automatically if/when your token expires. If you set it to + // false, it will not cache these settings, but re-authentication will not be + // possible. This setting defaults to false. + AllowReauth bool `yaml:"allow_reauth,omitempty" json:"allow_reauth,omitempty"` +} + +// Region represents a region included as part of cloud in clouds.yaml +// According to Python-based openstacksdk, this can be either a struct (as defined) +// or a plain string. Custom unmarshallers handle both cases. +type Region struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Values Cloud `yaml:"values,omitempty" json:"values,omitempty"` +} + +// UnmarshalJSON handles either a plain string acting as the Name property or +// a struct, mimicking the Python-based openstacksdk. +func (r *Region) UnmarshalJSON(data []byte) error { + var name string + if err := json.Unmarshal(data, &name); err == nil { + r.Name = name + return nil + } + + type region Region + var tmp region + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + r.Name = tmp.Name + r.Values = tmp.Values + + return nil +} + +// UnmarshalYAML handles either a plain string acting as the Name property or +// a struct, mimicking the Python-based openstacksdk. +func (r *Region) UnmarshalYAML(unmarshal func(interface{}) error) error { + var name string + if err := unmarshal(&name); err == nil { + r.Name = name + return nil + } + + type region Region + var tmp region + if err := unmarshal(&tmp); err != nil { + return err + } + r.Name = tmp.Name + r.Values = tmp.Values + + return nil +} + +// AuthType respresents a valid method of authentication. +type AuthType string + +const ( + // AuthPassword defines an unknown version of the password. + AuthPassword AuthType = "password" + // AuthToken defined an unknown version of the token. + AuthToken AuthType = "token" + + // AuthV2Password defines version 2 of the password. + AuthV2Password AuthType = "v2password" + // AuthV2Token defines version 2 of the token. + AuthV2Token AuthType = "v2token" + + // AuthV3Password defines version 3 of the password. + AuthV3Password AuthType = "v3password" + // AuthV3Token defines version 3 of the token. + AuthV3Token AuthType = "v3token" + + // AuthV3ApplicationCredential defines version 3 of the application credential. + AuthV3ApplicationCredential AuthType = "v3applicationcredential" +) diff --git a/internal/cloud/openstack/clouds/read.go b/internal/cloud/openstack/clouds/read.go new file mode 100644 index 000000000..1ffa4976a --- /dev/null +++ b/internal/cloud/openstack/clouds/read.go @@ -0,0 +1,59 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ +package clouds + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/mitchellh/go-homedir" + + "github.com/edgelesssys/constellation/v2/internal/file" +) + +// ReadCloudsYAML reads a clouds.yaml file and returns its contents. +func ReadCloudsYAML(fileHandler file.Handler, path string) (Clouds, error) { + // Order of operations as performed by the OpenStack CLI: + + // Define a search path for clouds.yaml: + // 1. If OS_CLIENT_CONFIG_FILE is set, use it as search path + // 2. Otherwise, use the following paths: + // - current directory + // - `openstack` directory under standard user config directory (e.g. ~/.config/openstack) + // - /etc/openstack (Unix only) + + var searchPaths []string + if path != "" { + expanded, err := homedir.Expand(path) + if err == nil { + searchPaths = append(searchPaths, expanded) + } else { + searchPaths = append(searchPaths, path) + } + } else if osClientConfigFile := os.Getenv("OS_CLIENT_CONFIG_FILE"); osClientConfigFile != "" { + searchPaths = append(searchPaths, filepath.Join(osClientConfigFile, "clouds.yaml")) + } else { + searchPaths = append(searchPaths, "clouds.yaml") + confDir, err := os.UserConfigDir() + if err != nil { + return Clouds{}, fmt.Errorf("getting user config directory: %w", err) + } + searchPaths = append(searchPaths, filepath.Join(confDir, "openstack", "clouds.yaml")) + if os.PathSeparator == '/' { + searchPaths = append(searchPaths, "/etc/openstack/clouds.yaml") + } + } + + var cloudsYAML Clouds + for _, path := range searchPaths { + if err := fileHandler.ReadYAML(path, &cloudsYAML); err == nil { + return cloudsYAML, nil + } + } + + return Clouds{}, fmt.Errorf("clouds.yaml not found in search paths: %v", searchPaths) +} diff --git a/internal/cloud/openstack/imds.go b/internal/cloud/openstack/imds.go index 01e31268b..50d255903 100644 --- a/internal/cloud/openstack/imds.go +++ b/internal/cloud/openstack/imds.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack @@ -22,21 +22,20 @@ import ( // documentation of OpenStack Metadata Service: https://docs.openstack.org/nova/rocky/user/metadata-service.html const ( - imdsMetaDataURL = "http://169.254.169.254/openstack/2018-08-27/meta_data.json" - imdsNetworkDataURL = "http://169.254.169.254/openstack/2018-08-27/network_data.json" - ec2ImdsBaseURL = "http://169.254.169.254/1.0/meta-data" - maxCacheAge = 12 * time.Hour + imdsMetaDataURL = "http://169.254.169.254/openstack/2018-08-27/meta_data.json" + imdsUserDataURL = "http://169.254.169.254/openstack/2018-08-27/user_data" + ec2ImdsBaseURL = "http://169.254.169.254/1.0/meta-data" + maxCacheAge = 12 * time.Hour ) type imdsClient struct { client httpClient - vpcIPCache string - vpcIPCacheTime time.Time - networkCache networkResponse - networkCacheTime time.Time - cache metadataResponse - cacheTime time.Time + vpcIPCache string + vpcIPCacheTime time.Time + cache metadataResponse + userDataCache userDataResponse + cacheTime time.Time } // providerID returns the provider ID of the instance the function is called from. @@ -131,60 +130,88 @@ func (c *imdsClient) role(ctx context.Context) (role.Role, error) { return role.FromString(c.cache.Tags.Role), nil } -func (c *imdsClient) authURL(ctx context.Context) (string, error) { - if c.timeForUpdate(c.cacheTime) || c.cache.Tags.AuthURL == "" { +func (c *imdsClient) loadBalancerEndpoint(ctx context.Context) (string, error) { + if c.timeForUpdate(c.cacheTime) || c.userDataCache.LoadBalancerEndpoint == "" { if err := c.update(ctx); err != nil { return "", err } } - if c.cache.Tags.AuthURL == "" { + if c.userDataCache.LoadBalancerEndpoint == "" { + return "", errors.New("unable to get load balancer endpoint") + } + + return c.userDataCache.LoadBalancerEndpoint, nil +} + +func (c *imdsClient) authURL(ctx context.Context) (string, error) { + if c.timeForUpdate(c.cacheTime) || c.userDataCache.AuthURL == "" { + if err := c.update(ctx); err != nil { + return "", err + } + } + + if c.userDataCache.AuthURL == "" { return "", errors.New("unable to get auth url") } - return c.cache.Tags.AuthURL, nil + return c.userDataCache.AuthURL, nil } func (c *imdsClient) userDomainName(ctx context.Context) (string, error) { - if c.timeForUpdate(c.cacheTime) || c.cache.Tags.UserDomainName == "" { + if c.timeForUpdate(c.cacheTime) || c.userDataCache.UserDomainName == "" { if err := c.update(ctx); err != nil { return "", err } } - if c.cache.Tags.UserDomainName == "" { + if c.userDataCache.UserDomainName == "" { return "", errors.New("unable to get user domain name") } - return c.cache.Tags.UserDomainName, nil + return c.userDataCache.UserDomainName, nil +} + +func (c *imdsClient) regionName(ctx context.Context) (string, error) { + if c.timeForUpdate(c.cacheTime) || c.userDataCache.RegionName == "" { + if err := c.update(ctx); err != nil { + return "", err + } + } + + if c.userDataCache.RegionName == "" { + return "", errors.New("unable to get user domain name") + } + + return c.userDataCache.RegionName, nil } func (c *imdsClient) username(ctx context.Context) (string, error) { - if c.timeForUpdate(c.cacheTime) || c.cache.Tags.Username == "" { + if c.timeForUpdate(c.cacheTime) || c.userDataCache.Username == "" { if err := c.update(ctx); err != nil { return "", err } } - if c.cache.Tags.Username == "" { + if c.userDataCache.Username == "" { return "", errors.New("unable to get token name") } - return c.cache.Tags.Username, nil + return c.userDataCache.Username, nil } func (c *imdsClient) password(ctx context.Context) (string, error) { - if c.timeForUpdate(c.cacheTime) || c.cache.Tags.Password == "" { + if c.timeForUpdate(c.cacheTime) || c.userDataCache.Password == "" { if err := c.update(ctx); err != nil { return "", err } } - if c.cache.Tags.Password == "" { + if c.userDataCache.Password == "" { return "", errors.New("unable to get token password") } - return c.cache.Tags.Password, nil + return c.userDataCache.Password, nil } // timeForUpdate checks whether an update is needed due to cache age. @@ -192,18 +219,41 @@ func (c *imdsClient) timeForUpdate(t time.Time) bool { return time.Since(t) > maxCacheAge } -// update updates instance metadata from the azure imds API. func (c *imdsClient) update(ctx context.Context) error { + if err := c.updateInstanceMetadata(ctx); err != nil { + return fmt.Errorf("updating instance metadata: %w", err) + } + if err := c.updateUserData(ctx); err != nil { + return fmt.Errorf("updating user data: %w", err) + } + c.cacheTime = time.Now() + return nil +} + +// update updates instance metadata from the azure imds API. +func (c *imdsClient) updateInstanceMetadata(ctx context.Context) error { resp, err := httpGet(ctx, c.client, imdsMetaDataURL) if err != nil { return err } var metadataResp metadataResponse if err := json.Unmarshal(resp, &metadataResp); err != nil { - return err + return fmt.Errorf("unmarshalling IMDS metadata response %q: %w", resp, err) } c.cache = metadataResp - c.cacheTime = time.Now() + return nil +} + +func (c *imdsClient) updateUserData(ctx context.Context) error { + resp, err := httpGet(ctx, c.client, imdsUserDataURL) + if err != nil { + return err + } + var userdataResp userDataResponse + if err := json.Unmarshal(resp, &userdataResp); err != nil { + return fmt.Errorf("unmarshalling IMDS user_data response %q: %w", resp, err) + } + c.userDataCache = userdataResp return nil } @@ -226,30 +276,6 @@ func (c *imdsClient) vpcIP(ctx context.Context) (string, error) { return c.vpcIPCache, nil } -func (c *imdsClient) networkIDs(ctx context.Context) ([]string, error) { - if c.timeForUpdate(c.networkCacheTime) || len(c.networkCache.Networks) == 0 { - resp, err := httpGet(ctx, c.client, imdsNetworkDataURL) - if err != nil { - return nil, err - } - var networkResp networkResponse - if err := json.Unmarshal(resp, &networkResp); err != nil { - return nil, err - } - c.networkCache = networkResp - c.networkCacheTime = time.Now() - } - - var networkIDs []string - for _, network := range c.networkCache.Networks { - if network.NetworkID == "" { - continue - } - networkIDs = append(networkIDs, network.NetworkID) - } - return networkIDs, nil -} - func httpGet(ctx context.Context, c httpClient, url string) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { @@ -257,7 +283,10 @@ func httpGet(ctx context.Context, c httpClient, url string) ([]byte, error) { } resp, err := c.Do(req) if err != nil { - return nil, err + return nil, fmt.Errorf("querying the OpenStack IMDS api failed for %q: %w", url, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("IMDS api might be broken for this server. Recreate the cluster if this issue persists. Querying the OpenStack IMDS api failed for %q with error code %d", url, resp.StatusCode) } defer resp.Body.Close() return io.ReadAll(resp.Body) @@ -275,20 +304,15 @@ type metadataTags struct { InitSecretHash string `json:"constellation-init-secret-hash,omitempty"` Role string `json:"constellation-role,omitempty"` UID string `json:"constellation-uid,omitempty"` - AuthURL string `json:"openstack-auth-url,omitempty"` - UserDomainName string `json:"openstack-user-domain-name,omitempty"` - Username string `json:"openstack-username,omitempty"` - Password string `json:"openstack-password,omitempty"` } -// networkResponse contains networkResponse with only the required values. -type networkResponse struct { - Networks []metadataNetwork `json:"networks,omitempty"` -} - -type metadataNetwork struct { - ID string `json:"id,omitempty"` - NetworkID string `json:"network_id,omitempty"` +type userDataResponse struct { + AuthURL string `json:"openstack-auth-url,omitempty"` + UserDomainName string `json:"openstack-user-domain-name,omitempty"` + RegionName string `json:"openstack-region-name,omitempty"` + Username string `json:"openstack-username,omitempty"` + Password string `json:"openstack-password,omitempty"` + LoadBalancerEndpoint string `json:"openstack-load-balancer-endpoint,omitempty"` } type httpClient interface { diff --git a/internal/cloud/openstack/imds_test.go b/internal/cloud/openstack/imds_test.go index 5d5e0d25a..f3a135bfa 100644 --- a/internal/cloud/openstack/imds_test.go +++ b/internal/cloud/openstack/imds_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack @@ -26,7 +26,7 @@ func TestProviderID(t *testing.T) { someErr := errors.New("failed") type testCase struct { - cache metadataResponse + cache any cacheTime time.Time newClient httpClientJSONCreateFunc wantResult string @@ -34,7 +34,7 @@ func TestProviderID(t *testing.T) { wantErr bool } - newTestCases := func(mResp1, mResp2 metadataResponse, expect1, expect2 string) map[string]testCase { + newTestCases := func(mResp1, mResp2 any, expect1, expect2 string) map[string]testCase { return map[string]testCase{ "cached": { cache: mResp1, @@ -43,30 +43,30 @@ func TestProviderID(t *testing.T) { wantCall: false, }, "from http": { - newClient: newStubHTTPClientJSONFunc(mResp1, nil), + newClient: newStubHTTPClientJSONFunc(mResp1, 200, nil), wantResult: expect1, wantCall: true, }, "cache outdated": { cache: mResp1, cacheTime: time.Now().AddDate(0, 0, -1), - newClient: newStubHTTPClientJSONFunc(mResp2, nil), + newClient: newStubHTTPClientJSONFunc(mResp2, 200, nil), wantResult: expect2, wantCall: true, }, "cache empty": { cacheTime: time.Now(), - newClient: newStubHTTPClientJSONFunc(mResp1, nil), + newClient: newStubHTTPClientJSONFunc(mResp1, 200, nil), wantResult: expect1, wantCall: true, }, "http error": { - newClient: newStubHTTPClientJSONFunc(metadataResponse{}, someErr), + newClient: newStubHTTPClientJSONFunc(metadataResponse{}, 200, someErr), wantCall: true, wantErr: true, }, "http empty response": { - newClient: newStubHTTPClientJSONFunc(metadataResponse{}, nil), + newClient: newStubHTTPClientJSONFunc(metadataResponse{}, 200, nil), wantCall: true, wantErr: true, }, @@ -120,32 +120,32 @@ func TestProviderID(t *testing.T) { "authURL": { method: (*imdsClient).authURL, testCases: newTestCases( - metadataResponse{Tags: metadataTags{AuthURL: "authURL1"}}, - metadataResponse{Tags: metadataTags{AuthURL: "authURL2"}}, + userDataResponse{AuthURL: "authURL1"}, + userDataResponse{AuthURL: "authURL2"}, "authURL1", "authURL2", ), }, "userDomainName": { method: (*imdsClient).userDomainName, testCases: newTestCases( - metadataResponse{Tags: metadataTags{UserDomainName: "userDomainName1"}}, - metadataResponse{Tags: metadataTags{UserDomainName: "userDomainName2"}}, + userDataResponse{UserDomainName: "userDomainName1"}, + userDataResponse{UserDomainName: "userDomainName2"}, "userDomainName1", "userDomainName2", ), }, "username": { method: (*imdsClient).username, testCases: newTestCases( - metadataResponse{Tags: metadataTags{Username: "username1"}}, - metadataResponse{Tags: metadataTags{Username: "username2"}}, + userDataResponse{Username: "username1"}, + userDataResponse{Username: "username2"}, "username1", "username2", ), }, "password": { method: (*imdsClient).password, testCases: newTestCases( - metadataResponse{Tags: metadataTags{Password: "password1"}}, - metadataResponse{Tags: metadataTags{Password: "password2"}}, + userDataResponse{Password: "password1"}, + userDataResponse{Password: "password2"}, "password1", "password2", ), }, @@ -162,13 +162,21 @@ func TestProviderID(t *testing.T) { if tc.newClient != nil { client = tc.newClient(require) } + var cache metadataResponse + var userDataCache userDataResponse + if _, ok := tc.cache.(metadataResponse); ok { + cache = tc.cache.(metadataResponse) + } else if _, ok := tc.cache.(userDataResponse); ok { + userDataCache = tc.cache.(userDataResponse) + } imds := &imdsClient{ - client: client, - cache: tc.cache, - cacheTime: tc.cacheTime, + client: client, + cache: cache, + userDataCache: userDataCache, + cacheTime: tc.cacheTime, } - result, err := tu.method(imds, context.Background()) + result, err := tu.method(imds, t.Context()) if tc.wantErr { assert.Error(err) @@ -207,30 +215,35 @@ func TestRole(t *testing.T) { wantCall: false, }, "from http": { - newClient: newStubHTTPClientJSONFunc(mResp1, nil), + newClient: newStubHTTPClientJSONFunc(mResp1, 200, nil), wantResult: expect1, wantCall: true, }, "cache outdated": { cache: mResp1, cacheTime: time.Now().AddDate(0, 0, -1), - newClient: newStubHTTPClientJSONFunc(mResp2, nil), + newClient: newStubHTTPClientJSONFunc(mResp2, 200, nil), wantResult: expect2, wantCall: true, }, "cache empty": { cacheTime: time.Now(), - newClient: newStubHTTPClientJSONFunc(mResp1, nil), + newClient: newStubHTTPClientJSONFunc(mResp1, 200, nil), wantResult: expect1, wantCall: true, }, "http error": { - newClient: newStubHTTPClientJSONFunc(metadataResponse{}, someErr), + newClient: newStubHTTPClientJSONFunc(metadataResponse{}, 200, someErr), + wantCall: true, + wantErr: true, + }, + "http status code 500": { + newClient: newStubHTTPClientJSONFunc(metadataResponse{}, 500, nil), wantCall: true, wantErr: true, }, "http empty response": { - newClient: newStubHTTPClientJSONFunc(metadataResponse{}, nil), + newClient: newStubHTTPClientJSONFunc(metadataResponse{}, 200, nil), wantCall: true, wantErr: true, }, @@ -251,7 +264,7 @@ func TestRole(t *testing.T) { cacheTime: tc.cacheTime, } - result, err := imds.role(context.Background()) + result, err := imds.role(t.Context()) if tc.wantErr { assert.Error(err) @@ -323,125 +336,7 @@ func TestVPCIP(t *testing.T) { vpcIPCacheTime: tc.cacheTime, } - result, err := imds.vpcIP(context.Background()) - - if tc.wantErr { - assert.Error(err) - } else { - assert.NoError(err) - assert.Equal(tc.wantResult, result) - if tc.client != nil { - assert.Equal(tc.wantCall, tc.client.called) - } - } - }) - } -} - -func TestNetworkIDs(t *testing.T) { - someErr := errors.New("failed") - - testCases := map[string]struct { - cache networkResponse - cacheTime time.Time - client *stubHTTPClient - wantResult []string - wantCall bool - wantErr bool - }{ - "cached": { - cache: networkResponse{Networks: []metadataNetwork{ - {ID: "net0", NetworkID: "0000000-00000-0000-0000-000000000000"}, - {ID: "net1", NetworkID: "1111111-11111-1111-1111-111111111111"}, - {ID: "invalid"}, - }}, - cacheTime: time.Now(), - wantResult: []string{ - "0000000-00000-0000-0000-000000000000", - "1111111-11111-1111-1111-111111111111", - }, - wantCall: false, - }, - "from http": { - client: &stubHTTPClient{ - response: ` - { - "networks": [ - { - "id": "net0", - "network_id": "0000000-00000-0000-0000-000000000000" - }, - { - "id": "net1", - "network_id": "1111111-11111-1111-1111-111111111111" - } - ] - }`, - }, - wantResult: []string{ - "0000000-00000-0000-0000-000000000000", - "1111111-11111-1111-1111-111111111111", - }, - wantCall: true, - }, - "cache outdated": { - cache: networkResponse{Networks: []metadataNetwork{ - {ID: "net0", NetworkID: "0000000-00000-0000-0000-000000000000"}, - }}, - cacheTime: time.Now().AddDate(0, 0, -1), - client: &stubHTTPClient{ - response: ` - { - "networks": [ - { - "id": "net1", - "network_id": "1111111-11111-1111-1111-111111111111" - } - ] - }`, - }, - wantResult: []string{"1111111-11111-1111-1111-111111111111"}, - wantCall: true, - }, - "cache empty": { - cacheTime: time.Now(), - client: &stubHTTPClient{ - response: ` - { - "networks": [ - { - "id": "net0", - "network_id": "0000000-00000-0000-0000-000000000000" - } - ] - }`, - }, - wantResult: []string{"0000000-00000-0000-0000-000000000000"}, - wantCall: true, - }, - "http error": { - client: &stubHTTPClient{err: someErr}, - wantCall: true, - wantErr: true, - }, - "http empty response": { - client: &stubHTTPClient{}, - wantCall: true, - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - - imds := &imdsClient{ - client: tc.client, - networkCache: tc.cache, - networkCacheTime: tc.cacheTime, - } - - result, err := imds.networkIDs(context.Background()) + result, err := imds.vpcIP(t.Context()) if tc.wantErr { assert.Error(err) @@ -486,15 +381,17 @@ type httpClientJSONCreateFunc func(r *require.Assertions) *stubHTTPClientJSON type stubHTTPClientJSON struct { require *require.Assertions - response metadataResponse + response any + code int err error called bool } -func newStubHTTPClientJSONFunc(response metadataResponse, err error) httpClientJSONCreateFunc { +func newStubHTTPClientJSONFunc(response any, statusCode int, err error) httpClientJSONCreateFunc { return func(r *require.Assertions) *stubHTTPClientJSON { return &stubHTTPClientJSON{ response: response, + code: statusCode, err: err, require: r, } @@ -505,16 +402,26 @@ func (c *stubHTTPClientJSON) Do(_ *http.Request) (*http.Response, error) { c.called = true body, err := json.Marshal(c.response) c.require.NoError(err) - return &http.Response{Body: io.NopCloser(bytes.NewReader(body))}, c.err + code := 200 + if c.code != 0 { + code = c.code + } + return &http.Response{StatusCode: code, Status: http.StatusText(code), Body: io.NopCloser(bytes.NewReader(body))}, c.err } type stubHTTPClient struct { response string + code int err error called bool } func (c *stubHTTPClient) Do(_ *http.Request) (*http.Response, error) { c.called = true - return &http.Response{Body: io.NopCloser(strings.NewReader(c.response))}, c.err + code := 200 + if c.code != 0 { + code = c.code + } + + return &http.Response{StatusCode: code, Status: http.StatusText(code), Body: io.NopCloser(strings.NewReader(c.response))}, c.err } diff --git a/internal/cloud/openstack/openstack.go b/internal/cloud/openstack/openstack.go index ced06f395..4fae03421 100644 --- a/internal/cloud/openstack/openstack.go +++ b/internal/cloud/openstack/openstack.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack import ( "context" - "errors" "fmt" "net/http" "net/netip" @@ -18,9 +17,10 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/role" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - "github.com/gophercloud/utils/openstack/clientconfig" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" + "github.com/gophercloud/utils/v2/openstack/clientconfig" ) const ( @@ -28,14 +28,14 @@ const ( microversion = "2.42" ) -// Cloud is the metadata client for OpenStack. -type Cloud struct { +// MetadataClient is the metadata client for OpenStack. +type MetadataClient struct { api serversAPI imds imdsAPI } // New creates a new OpenStack metadata client. -func New(ctx context.Context) (*Cloud, error) { +func New(ctx context.Context) (*MetadataClient, error) { imds := &imdsClient{client: &http.Client{}} authURL, err := imds.authURL(ctx) @@ -54,6 +54,10 @@ func New(ctx context.Context) (*Cloud, error) { if err != nil { return nil, fmt.Errorf("getting user domain name: %w", err) } + regionName, err := imds.regionName(ctx) + if err != nil { + return nil, fmt.Errorf("getting region name: %w", err) + } clientOpts := &clientconfig.ClientOpts{ AuthType: clientconfig.AuthV3Password, @@ -63,31 +67,32 @@ func New(ctx context.Context) (*Cloud, error) { Username: username, Password: password, }, + RegionName: regionName, } - serversClient, err := clientconfig.NewServiceClient("compute", clientOpts) + serversClient, err := clientconfig.NewServiceClient(ctx, "compute", clientOpts) if err != nil { return nil, fmt.Errorf("creating compute client: %w", err) } serversClient.Microversion = microversion - subnetsClient, err := clientconfig.NewServiceClient("network", clientOpts) + networksClient, err := clientconfig.NewServiceClient(ctx, "network", clientOpts) if err != nil { return nil, fmt.Errorf("creating network client: %w", err) } - subnetsClient.Microversion = microversion + networksClient.Microversion = microversion - return &Cloud{ + return &MetadataClient{ imds: imds, api: &apiClient{ - servers: serversClient, - subnets: subnetsClient, + servers: serversClient, + networks: networksClient, }, }, nil } // Self returns the metadata of the current instance. -func (c *Cloud) Self(ctx context.Context) (metadata.InstanceMetadata, error) { +func (c *MetadataClient) Self(ctx context.Context) (metadata.InstanceMetadata, error) { name, err := c.imds.name(ctx) if err != nil { return metadata.InstanceMetadata{}, fmt.Errorf("getting name: %w", err) @@ -114,7 +119,7 @@ func (c *Cloud) Self(ctx context.Context) (metadata.InstanceMetadata, error) { } // List returns the metadata of all instances belonging to the same Constellation cluster. -func (c *Cloud) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { +func (c *MetadataClient) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { uid, err := c.imds.uid(ctx) if err != nil { return nil, fmt.Errorf("getting uid: %w", err) @@ -122,12 +127,12 @@ func (c *Cloud) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { uidTag := fmt.Sprintf("constellation-uid-%s", uid) - subnet, err := c.getSubnetCIDR(uidTag) + subnet, err := c.getSubnetCIDR(ctx, uidTag) if err != nil { return nil, err } - srvs, err := c.getServers(uidTag) + srvs, err := c.getServers(ctx, uidTag) if err != nil { return nil, err } @@ -211,7 +216,7 @@ func (c *Cloud) List(ctx context.Context) ([]metadata.InstanceMetadata, error) { } // UID retrieves the UID of the constellation. -func (c *Cloud) UID(ctx context.Context) (string, error) { +func (c *MetadataClient) UID(ctx context.Context) (string, error) { uid, err := c.imds.uid(ctx) if err != nil { return "", fmt.Errorf("retrieving instance UID: %w", err) @@ -220,7 +225,7 @@ func (c *Cloud) UID(ctx context.Context) (string, error) { } // InitSecretHash retrieves the InitSecretHash of the current instance. -func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) { +func (c *MetadataClient) InitSecretHash(ctx context.Context) ([]byte, error) { initSecretHash, err := c.imds.initSecretHash(ctx) if err != nil { return nil, fmt.Errorf("retrieving init secret hash: %w", err) @@ -232,108 +237,52 @@ func (c *Cloud) InitSecretHash(ctx context.Context) ([]byte, error) { // For OpenStack, the load balancer is a floating ip attached to // a control plane node. // TODO(malt3): Rewrite to use real load balancer once it is available. -func (c *Cloud) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) { - host, err = c.getLoadBalancerHost(ctx) +func (c *MetadataClient) GetLoadBalancerEndpoint(ctx context.Context) (host, port string, err error) { + host, err = c.imds.loadBalancerEndpoint(ctx) if err != nil { - return "", "", fmt.Errorf("getting load balancer host: %w", err) + return "", "", fmt.Errorf("getting load balancer endpoint: %w", err) } return host, strconv.FormatInt(constants.KubernetesPort, 10), nil } -func (c *Cloud) getLoadBalancerHost(ctx context.Context) (string, error) { - uid, err := c.imds.uid(ctx) +func (c *MetadataClient) getSubnetCIDR(ctx context.Context, uidTag string) (netip.Prefix, error) { + listNetworksOpts := networks.ListOpts{Tags: uidTag} + networksPage, err := c.api.ListNetworks(listNetworksOpts).AllPages(ctx) if err != nil { - return "", fmt.Errorf("getting uid: %w", err) + return netip.Prefix{}, fmt.Errorf("listing networks: %w", err) } - - uidTag := fmt.Sprintf("constellation-uid-%s", uid) - - subnet, err := c.getSubnetCIDR(uidTag) + nets, err := networks.ExtractNetworks(networksPage) if err != nil { - return "", err + return netip.Prefix{}, fmt.Errorf("extracting networks: %w", err) + } + if len(nets) != 1 { + return netip.Prefix{}, fmt.Errorf("expected exactly one network, got %d", len(nets)) } - srvs, err := c.getServers(uidTag) - if err != nil { - return "", err - } - - for _, s := range srvs { - if s.Name == "" { - continue - } - if s.ID == "" { - continue - } - if s.Tags == nil { - continue - } - - subnetAddrs, err := parseSeverAddresses(s.Addresses) - if err != nil { - return "", fmt.Errorf("parsing server %q addresses: %w", s.Name, err) - } - - // In a best effort approach, we take the first fixed IPv4 address that is outside the subnet - // belonging to our cluster and assume it is the "load balancer" floating ip. - for _, serverSubnet := range subnetAddrs { - for _, addr := range serverSubnet.Addresses { - if addr.Type != floatingIP { - continue - } - - if addr.IPVersion != ipV4 { - continue - } - - if addr.IP == "" { - continue - } - - parsedAddr, err := netip.ParseAddr(addr.IP) - if err != nil { - continue - } - - if subnet.Contains(parsedAddr) { - continue - } - - return addr.IP, nil - } - } - } - - return "", errors.New("no load balancer endpoint found") -} - -// GetNetworkIDs returns the IDs of the networks the current instance is part of. -// This method is OpenStack specific. -func (c *Cloud) GetNetworkIDs(ctx context.Context) ([]string, error) { - networkIDs, err := c.imds.networkIDs(ctx) - if err != nil { - return nil, fmt.Errorf("retrieving network IDs: %w", err) - } - return networkIDs, nil -} - -func (c *Cloud) getSubnetCIDR(uidTag string) (netip.Prefix, error) { listSubnetsOpts := subnets.ListOpts{Tags: uidTag} - subnetsPage, err := c.api.ListSubnets(listSubnetsOpts).AllPages() + subnetsPage, err := c.api.ListSubnets(listSubnetsOpts).AllPages(ctx) if err != nil { return netip.Prefix{}, fmt.Errorf("listing subnets: %w", err) } - nets, err := subnets.ExtractSubnets(subnetsPage) + snets, err := subnets.ExtractSubnets(subnetsPage) if err != nil { return netip.Prefix{}, fmt.Errorf("extracting subnets: %w", err) } - if len(nets) != 1 { - return netip.Prefix{}, fmt.Errorf("expected exactly one subnet, got %d", len(nets)) + if len(snets) < 1 { + return netip.Prefix{}, fmt.Errorf("expected at least one subnet, got %d", len(snets)) } - cidr, err := netip.ParsePrefix(nets[0].CIDR) + var rawCIDR string + for _, n := range snets { + if n.Name == nets[0].Name { + rawCIDR = n.CIDR + break + } + } + + cidr, err := netip.ParsePrefix(rawCIDR) if err != nil { return netip.Prefix{}, fmt.Errorf("parsing subnet CIDR: %w", err) } @@ -341,9 +290,9 @@ func (c *Cloud) getSubnetCIDR(uidTag string) (netip.Prefix, error) { return cidr, nil } -func (c *Cloud) getServers(uidTag string) ([]servers.Server, error) { +func (c *MetadataClient) getServers(ctx context.Context, uidTag string) ([]servers.Server, error) { listServersOpts := servers.ListOpts{Tags: uidTag} - serversPage, err := c.api.ListServers(listServersOpts).AllPages() + serversPage, err := c.api.ListServers(listServersOpts).AllPages(ctx) if err != nil { return nil, fmt.Errorf("listing servers: %w", err) } diff --git a/internal/cloud/openstack/openstack_test.go b/internal/cloud/openstack/openstack_test.go index c369486d9..0b9ecbbf8 100644 --- a/internal/cloud/openstack/openstack_test.go +++ b/internal/cloud/openstack/openstack_test.go @@ -1,23 +1,23 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack import ( - "context" "errors" "fmt" "testing" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/role" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" - "github.com/gophercloud/gophercloud/pagination" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" + "github.com/gophercloud/gophercloud/v2/pagination" "github.com/stretchr/testify/assert" ) @@ -85,9 +85,9 @@ func TestSelf(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - c := &Cloud{imds: tc.imds} + c := &MetadataClient{imds: tc.imds} - got, err := c.Self(context.Background()) + got, err := c.Self(t.Context()) if tc.wantErr { assert.Error(err) @@ -175,7 +175,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("198.51.100.1", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, want: []metadata.InstanceMetadata{ { @@ -196,7 +197,8 @@ func TestList(t *testing.T) { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ serversPager: newSeverPager([]servers.Server{}, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -204,9 +206,17 @@ func TestList(t *testing.T) { imds: &stubIMDSClient{uidErr: someErr}, wantErr: true, }, + "list nets errors": { + imds: &stubIMDSClient{uidResult: "uid"}, + api: &stubServersClient{ + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, someErr), + }, + wantErr: true, + }, "list subnets error": { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), subnetsPager: stubPager{allPagesErr: someErr}, }, wantErr: true, @@ -214,16 +224,18 @@ func TestList(t *testing.T) { "extract subnets error": { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), subnetsPager: newSubnetPager([]subnets.Subnet{}, someErr), }, wantErr: true, }, - "multiple subnets error": { + "subnet name mismatch error": { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), subnetsPager: newSubnetPager([]subnets.Subnet{ - {CIDR: "192.0.2.0/24"}, - {CIDR: "198.51.100.0/24"}, + {Name: "othernet", CIDR: "192.0.2.0/24"}, + {Name: "yetanothernet", CIDR: "198.51.100.0/24"}, }, nil), }, wantErr: true, @@ -231,7 +243,8 @@ func TestList(t *testing.T) { "parse subnet error": { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "notAnIP"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "notAnIP"}}, nil), }, wantErr: true, }, @@ -239,7 +252,8 @@ func TestList(t *testing.T) { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ serversPager: stubPager{allPagesErr: someErr}, - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -247,7 +261,8 @@ func TestList(t *testing.T) { imds: &stubIMDSClient{uidResult: "uid"}, api: &stubServersClient{ serversPager: newSeverPager([]servers.Server{}, someErr), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -261,7 +276,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("192.0.2.5", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -275,7 +291,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("192.0.2.5", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -289,7 +306,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("192.0.2.5", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -304,7 +322,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("192.0.2.5", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -319,7 +338,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("192.0.2.5", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -334,7 +354,8 @@ func TestList(t *testing.T) { Addresses: map[string]any{"foo": "bar"}, }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -349,7 +370,8 @@ func TestList(t *testing.T) { Addresses: newTestAddrs("invalidIP", ""), }, }, nil), - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), + netsPager: newNetPager([]networks.Network{{Name: "mynet"}}, nil), + subnetsPager: newSubnetPager([]subnets.Subnet{{Name: "mynet", CIDR: "192.0.2.0/24"}}, nil), }, wantErr: true, }, @@ -359,9 +381,9 @@ func TestList(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - c := &Cloud{imds: tc.imds, api: tc.api} + c := &MetadataClient{imds: tc.imds, api: tc.api} - got, err := c.List(context.Background()) + got, err := c.List(t.Context()) if tc.wantErr { assert.Error(err) @@ -393,9 +415,9 @@ func TestUID(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - c := &Cloud{imds: tc.imds} + c := &MetadataClient{imds: tc.imds} - got, err := c.UID(context.Background()) + got, err := c.UID(t.Context()) if tc.wantErr { assert.Error(err) @@ -427,9 +449,9 @@ func TestInitSecretHash(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - c := &Cloud{imds: tc.imds} + c := &MetadataClient{imds: tc.imds} - got, err := c.InitSecretHash(context.Background()) + got, err := c.InitSecretHash(t.Context()) if tc.wantErr { assert.Error(err) @@ -442,247 +464,28 @@ func TestInitSecretHash(t *testing.T) { } func TestGetLoadBalancerEndpoint(t *testing.T) { - // newTestAddrs returns a set of raw server addresses as we would get from - // a ListServers call and as expected by the parseSeverAddresses function. - // The hardcoded addresses don't match what we are looking for. A valid - // address can be injected. You can pass a second valid address to test - // that the first valid one is chosen. - newTestAddrs := func(floatingIP1, floatingIP2 string, fixedIP1 string) map[string]any { - return map[string]any{ - "network1": []any{ - map[string]any{ - "addr": "192.0.2.2", - "version": 4, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - }, - "network2": []any{ - map[string]any{ - "addr": fixedIP1, - "version": 4, - "OS-EXT-IPS:type": "fixed", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - map[string]any{ - "addr": "2001:db8:3333:4444:5555:6666:7777:8888", - "version": 6, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - map[string]any{ - "addr": floatingIP1, - "version": 4, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - map[string]any{ - "addr": floatingIP2, - "version": 4, - "OS-EXT-IPS:type": "floating", - "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:0c:0c:0c", - }, - }, - } - } - testCases := map[string]struct { - imds *stubIMDSClient - api *stubServersClient - wantHost string - wantErr bool - }{ - "error returned from IMDS client": { - imds: &stubIMDSClient{uidErr: errors.New("failed")}, - wantErr: true, - }, - "error returned from getSubnetCIDR": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager(nil, errors.New("failed")), - }, - wantErr: true, - }, - "error returned from getServers": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager(nil, errors.New("failed")), - }, - wantErr: true, - }, - "sever with empty name skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "server with empty ID skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "sever with nil tags skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "server has invalid address": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("", "", "invalidIP"), - }, - }, nil), - }, - wantErr: true, - }, - "server without parseable addresses skipped": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: map[string]any{ - "somekey": "invalid", - }, - }, - }, nil), - }, - wantErr: true, - }, - "invalid endpoint returned from server addresses": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("invalidIP", "", "192.0.2.1"), - }, - }, nil), - }, - wantErr: true, - }, - "valid endpoint returned from server addresses not in subnet CIDR": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "", "192.0.2.1"), - }, - }, nil), - }, - wantHost: "198.51.100.0", - }, - "first valid endpoint returned from server addresses not in subnet CIDR": { - imds: &stubIMDSClient{}, - api: &stubServersClient{ - subnetsPager: newSubnetPager([]subnets.Subnet{{CIDR: "192.0.2.0/24"}}, nil), - serversPager: newSeverPager([]servers.Server{ - { - Name: "name1", - ID: "id1", - Tags: &[]string{"constellation-role-control-plane", "constellation-uid-7777"}, - Addresses: newTestAddrs("198.51.100.0", "198.51.100.1", "192.0.2.1"), - }, - }, nil), - }, - wantHost: "198.51.100.0", - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - - c := &Cloud{ - imds: tc.imds, - api: tc.api, - } - - gotHost, gotPort, err := c.GetLoadBalancerEndpoint(context.Background()) - - if tc.wantErr { - assert.Error(err) - } else { - assert.NoError(err) - assert.Equal(tc.wantHost, gotHost) - assert.Equal("6443", gotPort) - } - }) - } -} - -func TestGetNetworkIDs(t *testing.T) { - someErr := fmt.Errorf("failed") - - testCases := map[string]struct { - imds imdsAPI - want []string + imds *stubIMDSClient + want string wantErr bool }{ - "success": { - imds: &stubIMDSClient{ - networkIDsResult: []string{"id1", "id2"}, - }, - want: []string{"id1", "id2"}, - }, - "fail to get network IDs": { - imds: &stubIMDSClient{ - networkIDsErr: someErr, - }, + "error returned from IMDS client": { + imds: &stubIMDSClient{loadBalancerEndpointErr: errors.New("failed")}, wantErr: true, }, + "UID returned from IMDS client": { + imds: &stubIMDSClient{loadBalancerEndpointResult: "some.endpoint"}, + want: "some.endpoint", + }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) - c := &Cloud{imds: tc.imds} + c := &MetadataClient{imds: tc.imds} - got, err := c.GetNetworkIDs(context.Background()) + got, _, err := c.GetLoadBalancerEndpoint(t.Context()) if tc.wantErr { assert.Error(err) @@ -694,6 +497,24 @@ func TestGetNetworkIDs(t *testing.T) { } } +// newNetPager returns a network pager as we would get from a ListNetworks. +func newNetPager(nets []networks.Network, err error) stubPager { + return stubPager{ + page: networks.NetworkPage{ + LinkedPageBase: pagination.LinkedPageBase{ + PageResult: pagination.PageResult{ + Result: gophercloud.Result{ + Body: struct { + Networks []networks.Network `json:"networks"` + }{nets}, + Err: err, + }, + }, + }, + }, + } +} + // newSubnetPager returns a subnet pager as we would get from a ListSubnets. func newSubnetPager(nets []subnets.Subnet, err error) stubPager { return stubPager{ diff --git a/internal/cloud/openstack/plumbing.go b/internal/cloud/openstack/plumbing.go index fa304994e..f99bafe75 100644 --- a/internal/cloud/openstack/plumbing.go +++ b/internal/cloud/openstack/plumbing.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack diff --git a/internal/cloud/openstack/plumbing_test.go b/internal/cloud/openstack/plumbing_test.go index 5f13fd42c..40354da33 100644 --- a/internal/cloud/openstack/plumbing_test.go +++ b/internal/cloud/openstack/plumbing_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack diff --git a/internal/cloud/openstack/wrappers.go b/internal/cloud/openstack/wrappers.go index b8e78a0b5..002916541 100644 --- a/internal/cloud/openstack/wrappers.go +++ b/internal/cloud/openstack/wrappers.go @@ -1,26 +1,31 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package openstack import ( - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" - "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets" ) type apiClient struct { - servers *gophercloud.ServiceClient - subnets *gophercloud.ServiceClient + servers *gophercloud.ServiceClient + networks *gophercloud.ServiceClient } func (c *apiClient) ListServers(opts servers.ListOptsBuilder) pagerAPI { return servers.List(c.servers, opts) } -func (c *apiClient) ListSubnets(opts subnets.ListOpts) pagerAPI { - return subnets.List(c.subnets, opts) +func (c *apiClient) ListNetworks(opts networks.ListOptsBuilder) pagerAPI { + return networks.List(c.networks, opts) +} + +func (c *apiClient) ListSubnets(opts subnets.ListOpts) pagerAPI { + return subnets.List(c.networks, opts) } diff --git a/internal/cloud/qemu/BUILD.bazel b/internal/cloud/qemu/BUILD.bazel index 5144883a4..ec4e2eadf 100644 --- a/internal/cloud/qemu/BUILD.bazel +++ b/internal/cloud/qemu/BUILD.bazel @@ -2,10 +2,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "qemu", - srcs = [ - "logger.go", - "qemu.go", - ], + srcs = ["qemu.go"], importpath = "github.com/edgelesssys/constellation/v2/internal/cloud/qemu", visibility = ["//:__subpackages__"], deps = [ diff --git a/internal/cloud/qemu/logger.go b/internal/cloud/qemu/logger.go deleted file mode 100644 index 85846ed83..000000000 --- a/internal/cloud/qemu/logger.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package qemu - -import ( - "context" - "net/http" - "net/url" - "strings" -) - -// Logger is a Cloud Logger for QEMU. -type Logger struct{} - -// NewLogger creates a new Cloud Logger for QEMU. -func NewLogger() *Logger { - return &Logger{} -} - -// Disclose writes log information to QEMU's cloud log. -// This is done by sending a POST request to the QEMU's metadata endpoint. -func (l *Logger) Disclose(msg string) { - url := &url.URL{ - Scheme: "http", - Host: qemuMetadataEndpoint, - Path: "/log", - } - - req, _ := http.NewRequestWithContext(context.Background(), http.MethodPost, url.String(), strings.NewReader(msg)) - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) - if err == nil { - defer resp.Body.Close() - } -} - -// Close is a no-op. -func (l *Logger) Close() error { - return nil -} diff --git a/internal/cloud/qemu/qemu.go b/internal/cloud/qemu/qemu.go index 5451342ab..a68682501 100644 --- a/internal/cloud/qemu/qemu.go +++ b/internal/cloud/qemu/qemu.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/compatibility/compatibility.go b/internal/compatibility/compatibility.go index e089a7ddd..c1a4ec08e 100644 --- a/internal/compatibility/compatibility.go +++ b/internal/compatibility/compatibility.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/compatibility/compatibility_test.go b/internal/compatibility/compatibility_test.go index 35c4e7517..48da2d4b8 100644 --- a/internal/compatibility/compatibility_test.go +++ b/internal/compatibility/compatibility_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package compatibility diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index c6b99c514..8a96864c5 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "azure.go", "config.go", "config_doc.go", + "gcp.go", # keep "image_enterprise.go", # keep @@ -30,6 +31,7 @@ go_library( "//internal/config/imageversion", "//internal/config/instancetypes", "//internal/constants", + "//internal/encoding", "//internal/file", "//internal/role", "//internal/semver", @@ -61,6 +63,7 @@ go_test( "//internal/cloud/cloudprovider", "//internal/config/instancetypes", "//internal/constants", + "//internal/encoding", "//internal/file", "//internal/semver", "//internal/versions", diff --git a/internal/config/attestation.go b/internal/config/attestation.go index 7821a63b5..08f980681 100644 --- a/internal/config/attestation.go +++ b/internal/config/attestation.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -17,8 +17,12 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/variant" ) -// arkPEM is the PEM encoded AMD root key. Received from the AMD Key Distribution System API (KDS). -const arkPEM = `-----BEGIN CERTIFICATE-----\nMIIGYzCCBBKgAwIBAgIDAQAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAIC\nBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDAS\nBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEg\nQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZp\nY2VzMRIwEAYDVQQDDAlBUkstTWlsYW4wHhcNMjAxMDIyMTcyMzA1WhcNNDUxMDIy\nMTcyMzA1WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDAS\nBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5j\nZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLU1pbGFuMIICIjANBgkqhkiG\n9w0BAQEFAAOCAg8AMIICCgKCAgEA0Ld52RJOdeiJlqK2JdsVmD7FktuotWwX1fNg\nW41XY9Xz1HEhSUmhLz9Cu9DHRlvgJSNxbeYYsnJfvyjx1MfU0V5tkKiU1EesNFta\n1kTA0szNisdYc9isqk7mXT5+KfGRbfc4V/9zRIcE8jlHN61S1ju8X93+6dxDUrG2\nSzxqJ4BhqyYmUDruPXJSX4vUc01P7j98MpqOS95rORdGHeI52Naz5m2B+O+vjsC0\n60d37jY9LFeuOP4Meri8qgfi2S5kKqg/aF6aPtuAZQVR7u3KFYXP59XmJgtcog05\ngmI0T/OitLhuzVvpZcLph0odh/1IPXqx3+MnjD97A7fXpqGd/y8KxX7jksTEzAOg\nbKAeam3lm+3yKIcTYMlsRMXPcjNbIvmsBykD//xSniusuHBkgnlENEWx1UcbQQrs\n+gVDkuVPhsnzIRNgYvM48Y+7LGiJYnrmE8xcrexekBxrva2V9TJQqnN3Q53kt5vi\nQi3+gCfmkwC0F0tirIZbLkXPrPwzZ0M9eNxhIySb2npJfgnqz55I0u33wh4r0ZNQ\neTGfw03MBUtyuzGesGkcw+loqMaq1qR4tjGbPYxCvpCq7+OgpCCoMNit2uLo9M18\nfHz10lOMT8nWAUvRZFzteXCm+7PHdYPlmQwUw3LvenJ/ILXoQPHfbkH0CyPfhl1j\nWhJFZasCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSFrBrRQ/fI\nrFXUxR1BSKvVeErUUzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuG\nKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvTWlsYW4vY3JsMEYGCSqG\nSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZI\nAWUDBAICBQCiAwIBMKMDAgEBA4ICAQC6m0kDp6zv4Ojfgy+zleehsx6ol0ocgVel\nETobpx+EuCsqVFRPK1jZ1sp/lyd9+0fQ0r66n7kagRk4Ca39g66WGTJMeJdqYriw\nSTjjDCKVPSesWXYPVAyDhmP5n2v+BYipZWhpvqpaiO+EGK5IBP+578QeW/sSokrK\ndHaLAxG2LhZxj9aF73fqC7OAJZ5aPonw4RE299FVarh1Tx2eT3wSgkDgutCTB1Yq\nzT5DuwvAe+co2CIVIzMDamYuSFjPN0BCgojl7V+bTou7dMsqIu/TW/rPCX9/EUcp\nKGKqPQ3P+N9r1hjEFY1plBg93t53OOo49GNI+V1zvXPLI6xIFVsh+mto2RtgEX/e\npmMKTNN6psW88qg7c1hTWtN6MbRuQ0vm+O+/2tKBF2h8THb94OvvHHoFDpbCELlq\nHnIYhxy0YKXGyaW1NjfULxrrmxVW4wcn5E8GddmvNa6yYm8scJagEi13mhGu4Jqh\n3QU3sf8iUSUr09xQDwHtOQUVIqx4maBZPBtSMf+qUDtjXSSq8lfWcd8bLr9mdsUn\nJZJ0+tuPMKmBnSH860llKk+VpVQsgqbzDIvOLvD6W1Umq25boxCYJ+TuBoa4s+HH\nCViAvgT9kf/rBq1d+ivj6skkHxuzcxbk1xv6ZGxrteJxVH7KlX7YRdZ6eARKwLe4\nAFZEAwoKCQ==\n-----END CERTIFICATE-----\n` +const ( + // arkPEM is the PEM encoded AMD root key certificate. Received from the AMD Key Distribution System API (KDS). + arkPEM = `-----BEGIN CERTIFICATE-----\nMIIGYzCCBBKgAwIBAgIDAQAAMEYGCSqGSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAIC\nBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZIAWUDBAICBQCiAwIBMKMDAgEBMHsxFDAS\nBgNVBAsMC0VuZ2luZWVyaW5nMQswCQYDVQQGEwJVUzEUMBIGA1UEBwwLU2FudGEg\nQ2xhcmExCzAJBgNVBAgMAkNBMR8wHQYDVQQKDBZBZHZhbmNlZCBNaWNybyBEZXZp\nY2VzMRIwEAYDVQQDDAlBUkstTWlsYW4wHhcNMjAxMDIyMTcyMzA1WhcNNDUxMDIy\nMTcyMzA1WjB7MRQwEgYDVQQLDAtFbmdpbmVlcmluZzELMAkGA1UEBhMCVVMxFDAS\nBgNVBAcMC1NhbnRhIENsYXJhMQswCQYDVQQIDAJDQTEfMB0GA1UECgwWQWR2YW5j\nZWQgTWljcm8gRGV2aWNlczESMBAGA1UEAwwJQVJLLU1pbGFuMIICIjANBgkqhkiG\n9w0BAQEFAAOCAg8AMIICCgKCAgEA0Ld52RJOdeiJlqK2JdsVmD7FktuotWwX1fNg\nW41XY9Xz1HEhSUmhLz9Cu9DHRlvgJSNxbeYYsnJfvyjx1MfU0V5tkKiU1EesNFta\n1kTA0szNisdYc9isqk7mXT5+KfGRbfc4V/9zRIcE8jlHN61S1ju8X93+6dxDUrG2\nSzxqJ4BhqyYmUDruPXJSX4vUc01P7j98MpqOS95rORdGHeI52Naz5m2B+O+vjsC0\n60d37jY9LFeuOP4Meri8qgfi2S5kKqg/aF6aPtuAZQVR7u3KFYXP59XmJgtcog05\ngmI0T/OitLhuzVvpZcLph0odh/1IPXqx3+MnjD97A7fXpqGd/y8KxX7jksTEzAOg\nbKAeam3lm+3yKIcTYMlsRMXPcjNbIvmsBykD//xSniusuHBkgnlENEWx1UcbQQrs\n+gVDkuVPhsnzIRNgYvM48Y+7LGiJYnrmE8xcrexekBxrva2V9TJQqnN3Q53kt5vi\nQi3+gCfmkwC0F0tirIZbLkXPrPwzZ0M9eNxhIySb2npJfgnqz55I0u33wh4r0ZNQ\neTGfw03MBUtyuzGesGkcw+loqMaq1qR4tjGbPYxCvpCq7+OgpCCoMNit2uLo9M18\nfHz10lOMT8nWAUvRZFzteXCm+7PHdYPlmQwUw3LvenJ/ILXoQPHfbkH0CyPfhl1j\nWhJFZasCAwEAAaN+MHwwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSFrBrRQ/fI\nrFXUxR1BSKvVeErUUzAPBgNVHRMBAf8EBTADAQH/MDoGA1UdHwQzMDEwL6AtoCuG\nKWh0dHBzOi8va2RzaW50Zi5hbWQuY29tL3ZjZWsvdjEvTWlsYW4vY3JsMEYGCSqG\nSIb3DQEBCjA5oA8wDQYJYIZIAWUDBAICBQChHDAaBgkqhkiG9w0BAQgwDQYJYIZI\nAWUDBAICBQCiAwIBMKMDAgEBA4ICAQC6m0kDp6zv4Ojfgy+zleehsx6ol0ocgVel\nETobpx+EuCsqVFRPK1jZ1sp/lyd9+0fQ0r66n7kagRk4Ca39g66WGTJMeJdqYriw\nSTjjDCKVPSesWXYPVAyDhmP5n2v+BYipZWhpvqpaiO+EGK5IBP+578QeW/sSokrK\ndHaLAxG2LhZxj9aF73fqC7OAJZ5aPonw4RE299FVarh1Tx2eT3wSgkDgutCTB1Yq\nzT5DuwvAe+co2CIVIzMDamYuSFjPN0BCgojl7V+bTou7dMsqIu/TW/rPCX9/EUcp\nKGKqPQ3P+N9r1hjEFY1plBg93t53OOo49GNI+V1zvXPLI6xIFVsh+mto2RtgEX/e\npmMKTNN6psW88qg7c1hTWtN6MbRuQ0vm+O+/2tKBF2h8THb94OvvHHoFDpbCELlq\nHnIYhxy0YKXGyaW1NjfULxrrmxVW4wcn5E8GddmvNa6yYm8scJagEi13mhGu4Jqh\n3QU3sf8iUSUr09xQDwHtOQUVIqx4maBZPBtSMf+qUDtjXSSq8lfWcd8bLr9mdsUn\nJZJ0+tuPMKmBnSH860llKk+VpVQsgqbzDIvOLvD6W1Umq25boxCYJ+TuBoa4s+HH\nCViAvgT9kf/rBq1d+ivj6skkHxuzcxbk1xv6ZGxrteJxVH7KlX7YRdZ6eARKwLe4\nAFZEAwoKCQ==\n-----END CERTIFICATE-----\n` + // tdxRootPEM is the PEM encoded Intel TDX root key certificate. Receieved from the Intel Provisioning Certification Service (PCS). + tdxRootPEM = `-----BEGIN CERTIFICATE-----\nMIICjzCCAjSgAwIBAgIUImUM1lqdNInzg7SVUr9QGzknBqwwCgYIKoZIzj0EAwIw\naDEaMBgGA1UEAwwRSW50ZWwgU0dYIFJvb3QgQ0ExGjAYBgNVBAoMEUludGVsIENv\ncnBvcmF0aW9uMRQwEgYDVQQHDAtTYW50YSBDbGFyYTELMAkGA1UECAwCQ0ExCzAJ\nBgNVBAYTAlVTMB4XDTE4MDUyMTEwNDUxMFoXDTQ5MTIzMTIzNTk1OVowaDEaMBgG\nA1UEAwwRSW50ZWwgU0dYIFJvb3QgQ0ExGjAYBgNVBAoMEUludGVsIENvcnBvcmF0\naW9uMRQwEgYDVQQHDAtTYW50YSBDbGFyYTELMAkGA1UECAwCQ0ExCzAJBgNVBAYT\nAlVTMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEC6nEwMDIYZOj/iPWsCzaEKi7\n1OiOSLRFhWGjbnBVJfVnkY4u3IjkDYYL0MxO4mqsyYjlBalTVYxFP2sJBK5zlKOB\nuzCBuDAfBgNVHSMEGDAWgBQiZQzWWp00ifODtJVSv1AbOScGrDBSBgNVHR8ESzBJ\nMEegRaBDhkFodHRwczovL2NlcnRpZmljYXRlcy50cnVzdGVkc2VydmljZXMuaW50\nZWwuY29tL0ludGVsU0dYUm9vdENBLmRlcjAdBgNVHQ4EFgQUImUM1lqdNInzg7SV\nUr9QGzknBqwwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwCgYI\nKoZIzj0EAwIDSQAwRgIhAOW/5QkR+S9CiSDcNoowLuPRLsWGf/Yi7GSX94BgwTwg\nAiEA4J0lrHoMs+Xo5o/sX6O9QWxHRAvZUGOdRQ7cvqRXaqI=\n-----END CERTIFICATE-----\n` +) // AttestationCfg is the common interface for passing attestation configs. type AttestationCfg interface { @@ -44,8 +48,12 @@ func UnmarshalAttestationConfig(data []byte, attestVariant variant.Variant) (Att return unmarshalTypedConfig[*AzureSEVSNP](data) case variant.AzureTrustedLaunch{}: return unmarshalTypedConfig[*AzureTrustedLaunch](data) + case variant.AzureTDX{}: + return unmarshalTypedConfig[*AzureTDX](data) case variant.GCPSEVES{}: return unmarshalTypedConfig[*GCPSEVES](data) + case variant.GCPSEVSNP{}: + return unmarshalTypedConfig[*GCPSEVSNP](data) case variant.QEMUVTPM{}: return unmarshalTypedConfig[*QEMUVTPM](data) case variant.QEMUTDX{}: diff --git a/internal/config/attestation_test.go b/internal/config/attestation_test.go index e0e3492dc..a13562ac1 100644 --- a/internal/config/attestation_test.go +++ b/internal/config/attestation_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -41,6 +41,9 @@ func TestUnmarshalAttestationConfig(t *testing.T) { "GCPSEVES": { cfg: &GCPSEVES{Measurements: measurements.DefaultsFor(cloudprovider.GCP, variant.GCPSEVES{})}, }, + "GCPSEVSNP": { + cfg: DefaultForGCPSEVSNP(), + }, "QEMUVTPM": { cfg: &QEMUVTPM{Measurements: measurements.DefaultsFor(cloudprovider.QEMU, variant.QEMUVTPM{})}, }, diff --git a/internal/config/attestationversion.go b/internal/config/attestationversion.go index a7949c5c3..c64025ca0 100644 --- a/internal/config/attestationversion.go +++ b/internal/config/attestationversion.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -9,47 +9,49 @@ package config import ( "encoding/json" "fmt" - "math" - "strconv" "strings" + + "github.com/edgelesssys/constellation/v2/internal/encoding" ) -const placeholderVersionValue = 0 +type versionValue interface { + encoding.HexBytes | uint8 | uint16 +} + +func placeholderVersionValue[T versionValue]() T { + var placeholder T + return placeholder +} // NewLatestPlaceholderVersion returns the latest version with a placeholder version value. -func NewLatestPlaceholderVersion() AttestationVersion { - return AttestationVersion{ - Value: placeholderVersionValue, +func NewLatestPlaceholderVersion[T versionValue]() AttestationVersion[T] { + return AttestationVersion[T]{ + Value: placeholderVersionValue[T](), WantLatest: true, } } -// AttestationVersion is a type that represents a version of a SNP. -type AttestationVersion struct { - Value uint8 +// AttestationVersion holds version information. +type AttestationVersion[T versionValue] struct { + Value T WantLatest bool } -// MarshalYAML implements a custom marshaller to resolve "latest" values. -func (v AttestationVersion) MarshalYAML() (any, error) { +// MarshalYAML implements a custom marshaller to write "latest" as the type's value, if set. +func (v AttestationVersion[T]) MarshalYAML() (any, error) { if v.WantLatest { return "latest", nil } return v.Value, nil } -// UnmarshalYAML implements a custom unmarshaller to resolve "atest" values. -func (v *AttestationVersion) UnmarshalYAML(unmarshal func(any) error) error { - var rawUnmarshal string - if err := unmarshal(&rawUnmarshal); err != nil { - return fmt.Errorf("raw unmarshal: %w", err) - } - - return v.parseRawUnmarshal(rawUnmarshal) +// UnmarshalYAML implements a custom unmarshaller to resolve "latest" values. +func (v *AttestationVersion[T]) UnmarshalYAML(unmarshal func(any) error) error { + return v.unmarshal(unmarshal) } -// MarshalJSON implements a custom marshaller to resolve "latest" values. -func (v AttestationVersion) MarshalJSON() ([]byte, error) { +// MarshalJSON implements a custom marshaller to write "latest" as the type's value, if set. +func (v AttestationVersion[T]) MarshalJSON() ([]byte, error) { if v.WantLatest { return json.Marshal("latest") } @@ -57,39 +59,31 @@ func (v AttestationVersion) MarshalJSON() ([]byte, error) { } // UnmarshalJSON implements a custom unmarshaller to resolve "latest" values. -func (v *AttestationVersion) UnmarshalJSON(data []byte) (err error) { - // JSON has two distinct ways to represent numbers and strings. - // This means we cannot simply unmarshal to string, like with YAML. - // Unmarshalling to `any` causes Go to unmarshal numbers to float64. - // Therefore, try to unmarshal to string, and then to int, instead of using type assertions. +func (v *AttestationVersion[T]) UnmarshalJSON(data []byte) (err error) { + return v.unmarshal(func(a any) error { + return json.Unmarshal(data, a) + }) +} + +// unmarshal takes care of unmarshalling the value from YAML or JSON. +func (v *AttestationVersion[T]) unmarshal(unmarshal func(any) error) error { + // Start by trying to unmarshal to the distinct type + var distinctType T + if err := unmarshal(&distinctType); err == nil { + v.Value = distinctType + return nil + } + var unmarshalString string - if err := json.Unmarshal(data, &unmarshalString); err != nil { - var unmarshalInt int64 - if err := json.Unmarshal(data, &unmarshalInt); err != nil { - return fmt.Errorf("unable to unmarshal to string or int: %w", err) - } - unmarshalString = strconv.FormatInt(unmarshalInt, 10) + if err := unmarshal(&unmarshalString); err != nil { + return fmt.Errorf("failed unmarshalling to %T or string: %w", distinctType, err) } - return v.parseRawUnmarshal(unmarshalString) -} - -func (v *AttestationVersion) parseRawUnmarshal(str string) error { - if strings.HasPrefix(str, "0") && len(str) != 1 { - return fmt.Errorf("no format with prefixed 0 (octal, hexadecimal) allowed: %s", str) - } - if strings.ToLower(str) == "latest" { + if strings.ToLower(unmarshalString) == "latest" { v.WantLatest = true - v.Value = placeholderVersionValue - } else { - ui, err := strconv.ParseUint(str, 10, 8) - if err != nil { - return fmt.Errorf("invalid version value: %s", str) - } - if ui > math.MaxUint8 { - return fmt.Errorf("integer value is out ouf uint8 range: %d", ui) - } - v.Value = uint8(ui) + v.Value = placeholderVersionValue[T]() + return nil } - return nil + + return fmt.Errorf("failed unmarshalling to %T or string: invalid value: %s", distinctType, unmarshalString) } diff --git a/internal/config/attestationversion_test.go b/internal/config/attestationversion_test.go index 52d68e2a8..c731831d2 100644 --- a/internal/config/attestationversion_test.go +++ b/internal/config/attestationversion_test.go @@ -1,210 +1,313 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config import ( + "bytes" "encoding/json" "testing" - "github.com/stretchr/testify/require" + "github.com/edgelesssys/constellation/v2/internal/encoding" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" ) func TestVersionMarshalYAML(t *testing.T) { - tests := map[string]struct { - sut AttestationVersion + testCasesUint8 := map[string]struct { + sut AttestationVersion[uint8] want string }{ - "isLatest resolves to latest": { - sut: AttestationVersion{ + "version with latest writes latest": { + sut: AttestationVersion[uint8]{ Value: 1, WantLatest: true, }, want: "latest\n", }, - "value 5 resolves to 5": { - sut: AttestationVersion{ + "value 5 writes 5": { + sut: AttestationVersion[uint8]{ Value: 5, WantLatest: false, }, want: "5\n", }, } - for name, tc := range tests { + for name, tc := range testCasesUint8 { t.Run(name, func(t *testing.T) { - require := require.New(t) + assert := assert.New(t) bt, err := yaml.Marshal(tc.sut) - require.NoError(err) - require.Equal(tc.want, string(bt)) + assert.NoError(err) + assert.Equal(tc.want, string(bt)) + }) + } + + testCasesUint16 := map[string]struct { + sut AttestationVersion[uint16] + want string + }{ + "version with latest writes latest": { + sut: AttestationVersion[uint16]{ + Value: 1, + WantLatest: true, + }, + want: "latest\n", + }, + "value 5 writes 5": { + sut: AttestationVersion[uint16]{ + Value: 5, + WantLatest: false, + }, + want: "5\n", + }, + } + for name, tc := range testCasesUint16 { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + bt, err := yaml.Marshal(tc.sut) + assert.NoError(err) + assert.Equal(tc.want, string(bt)) + }) + } + + testCasesHexBytes := map[string]struct { + sut AttestationVersion[encoding.HexBytes] + want string + }{ + "version with latest writes latest": { + sut: AttestationVersion[encoding.HexBytes]{ + Value: encoding.HexBytes(bytes.Repeat([]byte("0"), 16)), + WantLatest: true, + }, + want: "latest\n", + }, + "value 5 writes 5": { + sut: AttestationVersion[encoding.HexBytes]{ + Value: encoding.HexBytes(bytes.Repeat([]byte("A"), 16)), + WantLatest: false, + }, + want: "\"41414141414141414141414141414141\"\n", + }, + } + for name, tc := range testCasesHexBytes { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + bt, err := yaml.Marshal(tc.sut) + assert.NoError(err) + assert.Equal(tc.want, string(bt)) }) } } -func TestVersionUnmarshalYAML(t *testing.T) { - tests := map[string]struct { - sut string - want AttestationVersion - wantErr bool +func TestVersionUnmarshal(t *testing.T) { + testCasesUint8 := map[string]struct { + yamlData string + jsonData string + want AttestationVersion[uint8] + wantErr bool }{ "latest resolves to isLatest": { - sut: "latest", - want: AttestationVersion{ + yamlData: "latest", + jsonData: "\"latest\"", + want: AttestationVersion[uint8]{ Value: 0, WantLatest: true, }, wantErr: false, }, "1 resolves to value 1": { - sut: "1", - want: AttestationVersion{ + yamlData: "1", + jsonData: "1", + want: AttestationVersion[uint8]{ Value: 1, WantLatest: false, }, wantErr: false, }, "max uint8+1 errors": { - sut: "256", - wantErr: true, + yamlData: "256", + jsonData: "256", + wantErr: true, }, "-1 errors": { - sut: "-1", - wantErr: true, - }, - "2.6 errors": { - sut: "2.6", - wantErr: true, - }, - "2.0 errors": { - sut: "2.0", - wantErr: true, - }, - "hex format is invalid": { - sut: "0x10", - wantErr: true, - }, - "octal format is invalid": { - sut: "010", - wantErr: true, + yamlData: "-1", + jsonData: "-1", + wantErr: true, }, "0 resolves to value 0": { - sut: "0", - want: AttestationVersion{ + yamlData: "0", + jsonData: "0", + want: AttestationVersion[uint8]{ Value: 0, WantLatest: false, }, }, - "00 errors": { - sut: "00", - wantErr: true, - }, } - for name, tc := range tests { + for name, tc := range testCasesUint8 { t.Run(name, func(t *testing.T) { - require := require.New(t) + assert := assert.New(t) - var sut AttestationVersion - err := yaml.Unmarshal([]byte(tc.sut), &sut) - if tc.wantErr { - require.Error(err) - return + { + var sut AttestationVersion[uint8] + err := yaml.Unmarshal([]byte(tc.yamlData), &sut) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(tc.want, sut) + } + } + + { + var sut AttestationVersion[uint8] + err := json.Unmarshal([]byte(tc.jsonData), &sut) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(tc.want, sut) + } } - require.NoError(err) - require.Equal(tc.want, sut) }) } -} -func TestVersionUnmarshalJSON(t *testing.T) { - tests := map[string]struct { - sut string - want AttestationVersion - wantErr bool + testCasesUint16 := map[string]struct { + yamlData string + jsonData string + want AttestationVersion[uint16] + wantErr bool }{ "latest resolves to isLatest": { - sut: `"latest"`, - want: AttestationVersion{ + yamlData: "latest", + jsonData: "\"latest\"", + want: AttestationVersion[uint16]{ Value: 0, WantLatest: true, }, + wantErr: false, }, "1 resolves to value 1": { - sut: "1", - want: AttestationVersion{ + yamlData: "1", + jsonData: "1", + want: AttestationVersion[uint16]{ Value: 1, WantLatest: false, }, + wantErr: false, }, - "quoted number resolves to value": { - sut: `"1"`, - want: AttestationVersion{ - Value: 1, - WantLatest: false, - }, - }, - "quoted float errors": { - sut: `"1.0"`, - wantErr: true, - }, - "max uint8+1 errors": { - sut: "256", - wantErr: true, + "max uint16+1 errors": { + yamlData: "65536", + jsonData: "65536", + wantErr: true, }, "-1 errors": { - sut: "-1", - wantErr: true, - }, - "2.6 errors": { - sut: "2.6", - wantErr: true, - }, - "2.0 errors": { - sut: "2.0", - wantErr: true, - }, - "hex format is invalid": { - sut: "0x10", - wantErr: true, - }, - "octal format is invalid": { - sut: "010", - wantErr: true, + yamlData: "-1", + jsonData: "-1", + wantErr: true, }, "0 resolves to value 0": { - sut: "0", - want: AttestationVersion{ + yamlData: "0", + jsonData: "0", + want: AttestationVersion[uint16]{ Value: 0, WantLatest: false, }, }, - "quoted 0 resolves to value 0": { - sut: `"0"`, - want: AttestationVersion{ - Value: 0, - WantLatest: false, - }, - }, - "00 errors": { - sut: "00", - wantErr: true, - }, } - for name, tc := range tests { + for name, tc := range testCasesUint16 { t.Run(name, func(t *testing.T) { - require := require.New(t) + assert := assert.New(t) - var sut AttestationVersion - err := json.Unmarshal([]byte(tc.sut), &sut) - if tc.wantErr { - require.Error(err) - return + { + var sut AttestationVersion[uint16] + err := yaml.Unmarshal([]byte(tc.yamlData), &sut) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(tc.want, sut) + } + } + + { + var sut AttestationVersion[uint16] + err := json.Unmarshal([]byte(tc.jsonData), &sut) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(tc.want, sut) + } + } + }) + } + + testCasesHexBytes := map[string]struct { + yamlData string + jsonData string + want AttestationVersion[encoding.HexBytes] + wantErr bool + }{ + "latest resolves to isLatest": { + yamlData: "latest", + jsonData: "\"latest\"", + want: AttestationVersion[encoding.HexBytes]{ + Value: encoding.HexBytes(nil), + WantLatest: true, + }, + wantErr: false, + }, + "hex string resolves to correctly": { + yamlData: "41414141414141414141414141414141", + jsonData: "\"41414141414141414141414141414141\"", + want: AttestationVersion[encoding.HexBytes]{ + Value: encoding.HexBytes(bytes.Repeat([]byte("A"), 16)), + WantLatest: false, + }, + wantErr: false, + }, + "invalid hex string": { + yamlData: "GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG", + jsonData: "\"GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG\"", + wantErr: true, + }, + "non hex data": { + yamlData: "-15", + jsonData: "-15", + wantErr: true, + }, + } + for name, tc := range testCasesHexBytes { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + { + var sut AttestationVersion[encoding.HexBytes] + err := yaml.Unmarshal([]byte(tc.yamlData), &sut) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(tc.want, sut) + } + } + + { + var sut AttestationVersion[encoding.HexBytes] + err := json.Unmarshal([]byte(tc.jsonData), &sut) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + assert.Equal(tc.want, sut) + } } - require.NoError(err) - require.Equal(tc.want, sut) }) } } diff --git a/internal/config/aws.go b/internal/config/aws.go index f10fd4f86..43594342e 100644 --- a/internal/config/aws.go +++ b/internal/config/aws.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -16,14 +16,16 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" ) +var _ svnResolveMarshaller = &AWSSEVSNP{} + // DefaultForAWSSEVSNP provides a valid default configuration for AWS SEV-SNP attestation. func DefaultForAWSSEVSNP() *AWSSEVSNP { return &AWSSEVSNP{ Measurements: measurements.DefaultsFor(cloudprovider.AWS, variant.AWSSEVSNP{}), - BootloaderVersion: NewLatestPlaceholderVersion(), - TEEVersion: NewLatestPlaceholderVersion(), - SNPVersion: NewLatestPlaceholderVersion(), - MicrocodeVersion: NewLatestPlaceholderVersion(), + BootloaderVersion: NewLatestPlaceholderVersion[uint8](), + TEEVersion: NewLatestPlaceholderVersion[uint8](), + SNPVersion: NewLatestPlaceholderVersion[uint8](), + MicrocodeVersion: NewLatestPlaceholderVersion[uint8](), AMDRootKey: mustParsePEM(arkPEM), } } @@ -61,6 +63,15 @@ func (c AWSSEVSNP) EqualTo(other AttestationCfg) (bool, error) { return measurementsEqual && bootloaderEqual && teeEqual && snpEqual && microcodeEqual && rootKeyEqual && signingKeyEqual, nil } +func (c *AWSSEVSNP) getToMarshallLatestWithResolvedVersions() AttestationCfg { + cp := *c + cp.BootloaderVersion.WantLatest = false + cp.TEEVersion.WantLatest = false + cp.SNPVersion.WantLatest = false + cp.MicrocodeVersion.WantLatest = false + return &cp +} + // FetchAndSetLatestVersionNumbers fetches the latest version numbers from the configapi and sets them. func (c *AWSSEVSNP) FetchAndSetLatestVersionNumbers(ctx context.Context, fetcher attestationconfigapi.Fetcher) error { // Only talk to the API if at least one version number is set to latest. @@ -68,7 +79,7 @@ func (c *AWSSEVSNP) FetchAndSetLatestVersionNumbers(ctx context.Context, fetcher return nil } - versions, err := fetcher.FetchSEVSNPVersionLatest(ctx, variant.AWSSEVSNP{}) + versions, err := fetcher.FetchLatestVersion(ctx, variant.AWSSEVSNP{}) if err != nil { return fmt.Errorf("fetching latest TCB versions from configapi: %w", err) } diff --git a/internal/config/azure.go b/internal/config/azure.go index c6690e972..3631b52c8 100644 --- a/internal/config/azure.go +++ b/internal/config/azure.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -15,6 +15,12 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/encoding" +) + +var ( + _ svnResolveMarshaller = &AzureSEVSNP{} + _ svnResolveMarshaller = &AzureTDX{} ) // DefaultForAzureSEVSNP returns the default configuration for Azure SEV-SNP attestation. @@ -22,10 +28,10 @@ import ( func DefaultForAzureSEVSNP() *AzureSEVSNP { return &AzureSEVSNP{ Measurements: measurements.DefaultsFor(cloudprovider.Azure, variant.AzureSEVSNP{}), - BootloaderVersion: NewLatestPlaceholderVersion(), - TEEVersion: NewLatestPlaceholderVersion(), - SNPVersion: NewLatestPlaceholderVersion(), - MicrocodeVersion: NewLatestPlaceholderVersion(), + BootloaderVersion: NewLatestPlaceholderVersion[uint8](), + TEEVersion: NewLatestPlaceholderVersion[uint8](), + SNPVersion: NewLatestPlaceholderVersion[uint8](), + MicrocodeVersion: NewLatestPlaceholderVersion[uint8](), FirmwareSignerConfig: SNPFirmwareSignerConfig{ AcceptedKeyDigests: idkeydigest.DefaultList(), EnforcementPolicy: idkeydigest.MAAFallback, @@ -74,7 +80,7 @@ func (c *AzureSEVSNP) FetchAndSetLatestVersionNumbers(ctx context.Context, fetch return nil } - versions, err := fetcher.FetchSEVSNPVersionLatest(ctx, variant.AzureSEVSNP{}) + versions, err := fetcher.FetchLatestVersion(ctx, variant.AzureSEVSNP{}) if err != nil { return fmt.Errorf("fetching latest TCB versions from configapi: %w", err) } @@ -98,6 +104,15 @@ func (c *AzureSEVSNP) mergeWithLatestVersion(latest attestationconfigapi.SEVSNPV } } +func (c *AzureSEVSNP) getToMarshallLatestWithResolvedVersions() AttestationCfg { + cp := *c + cp.BootloaderVersion.WantLatest = false + cp.TEEVersion.WantLatest = false + cp.SNPVersion.WantLatest = false + cp.MicrocodeVersion.WantLatest = false + return &cp +} + // GetVariant returns azure-trusted-launch as the variant. func (AzureTrustedLaunch) GetVariant() variant.Variant { return variant.AzureTrustedLaunch{} @@ -121,3 +136,86 @@ func (c AzureTrustedLaunch) EqualTo(other AttestationCfg) (bool, error) { } return c.Measurements.EqualTo(otherCfg.Measurements), nil } + +// DefaultForAzureTDX returns the default configuration for Azure TDX attestation. +func DefaultForAzureTDX() *AzureTDX { + return &AzureTDX{ + Measurements: measurements.DefaultsFor(cloudprovider.Azure, variant.AzureTDX{}), + // TODO(AB#3798): Enable latest versioning for Azure TDX + QESVN: NewLatestPlaceholderVersion[uint16](), + PCESVN: NewLatestPlaceholderVersion[uint16](), + TEETCBSVN: NewLatestPlaceholderVersion[encoding.HexBytes](), + QEVendorID: NewLatestPlaceholderVersion[encoding.HexBytes](), + // Don't set a default for MRSEAM as it effectively prevents upgrading the SEAM module + // Quote verification still makes sure the module comes from Intel (through MRSIGNERSEAM), and is not of a lower version than expected + // MRSeam: nil, + XFAM: NewLatestPlaceholderVersion[encoding.HexBytes](), + + IntelRootKey: mustParsePEM(tdxRootPEM), + } +} + +// GetVariant returns azure-tdx as the variant. +func (AzureTDX) GetVariant() variant.Variant { + return variant.AzureTDX{} +} + +// GetMeasurements returns the measurements used for attestation. +func (c AzureTDX) GetMeasurements() measurements.M { + return c.Measurements +} + +// SetMeasurements updates a config's measurements using the given measurements. +func (c *AzureTDX) SetMeasurements(m measurements.M) { + c.Measurements = m +} + +// EqualTo returns true if the config is equal to the given config. +func (c AzureTDX) EqualTo(other AttestationCfg) (bool, error) { + otherCfg, ok := other.(*AzureTDX) + if !ok { + return false, fmt.Errorf("cannot compare %T with %T", c, other) + } + return c.Measurements.EqualTo(otherCfg.Measurements), nil +} + +// FetchAndSetLatestVersionNumbers fetches the latest version numbers from the configapi and sets them. +func (c *AzureTDX) FetchAndSetLatestVersionNumbers(ctx context.Context, fetcher attestationconfigapi.Fetcher) error { + // Only talk to the API if at least one version number is set to latest. + if !(c.PCESVN.WantLatest || c.QESVN.WantLatest || c.TEETCBSVN.WantLatest || c.QEVendorID.WantLatest || c.XFAM.WantLatest) { + return nil + } + + versions, err := fetcher.FetchLatestVersion(ctx, variant.AzureTDX{}) + if err != nil { + return fmt.Errorf("fetching latest TCB versions from configapi: %w", err) + } + + // set values and keep WantLatest flag + if c.PCESVN.WantLatest { + c.PCESVN.Value = versions.PCESVN + } + if c.QESVN.WantLatest { + c.QESVN.Value = versions.QESVN + } + if c.TEETCBSVN.WantLatest { + c.TEETCBSVN.Value = versions.TEETCBSVN[:] + } + if c.QEVendorID.WantLatest { + c.QEVendorID.Value = versions.QEVendorID[:] + } + if c.XFAM.WantLatest { + c.XFAM.Value = versions.XFAM[:] + } + return nil +} + +func (c *AzureTDX) getToMarshallLatestWithResolvedVersions() AttestationCfg { + cp := *c + cp.PCESVN.WantLatest = false + cp.QESVN.WantLatest = false + cp.TEETCBSVN.WantLatest = false + cp.QEVendorID.WantLatest = false + cp.XFAM.WantLatest = false + return &cp +} diff --git a/internal/config/config.go b/internal/config/config.go index 2025bbbf9..4397fa3fa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // This binary can be build from siderolabs/talos projects. Located at: @@ -41,6 +41,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/config/imageversion" "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/encoding" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" @@ -88,14 +89,17 @@ type Config struct { // The Kubernetes Service CIDR to be used for the cluster. This value will only be used during the first initialization of the Constellation. ServiceCIDR string `yaml:"serviceCIDR" validate:"omitempty,cidrv4"` // description: | + // Additional tags that are applied to created resources. + Tags cloudprovider.Tags `yaml:"tags" validate:"omitempty"` + // description: | // Supported cloud providers and their specific configurations. - Provider ProviderConfig `yaml:"provider" validate:"dive"` + Provider ProviderConfig `yaml:"provider"` // description: | // Node groups to be created in the cluster. NodeGroups map[string]NodeGroup `yaml:"nodeGroups" validate:"required,dive"` // description: | // Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation - Attestation AttestationConfig `yaml:"attestation" validate:"dive"` + Attestation AttestationConfig `yaml:"attestation"` } // ProviderConfig are cloud-provider specific configuration values used by the CLI. @@ -104,19 +108,19 @@ type Config struct { type ProviderConfig struct { // description: | // Configuration for AWS as provider. - AWS *AWSConfig `yaml:"aws,omitempty" validate:"omitempty,dive"` + AWS *AWSConfig `yaml:"aws,omitempty" validate:"omitempty"` // description: | // Configuration for Azure as provider. - Azure *AzureConfig `yaml:"azure,omitempty" validate:"omitempty,dive"` + Azure *AzureConfig `yaml:"azure,omitempty" validate:"omitempty"` // description: | // Configuration for Google Cloud as provider. - GCP *GCPConfig `yaml:"gcp,omitempty" validate:"omitempty,dive"` + GCP *GCPConfig `yaml:"gcp,omitempty" validate:"omitempty"` // description: | // Configuration for OpenStack as provider. - OpenStack *OpenStackConfig `yaml:"openstack,omitempty" validate:"omitempty,dive"` + OpenStack *OpenStackConfig `yaml:"openstack,omitempty" validate:"omitempty"` // description: | // Configuration for QEMU as provider. - QEMU *QEMUConfig `yaml:"qemu,omitempty" validate:"omitempty,dive"` + QEMU *QEMUConfig `yaml:"qemu,omitempty" validate:"omitempty"` } // AWSConfig are AWS specific configuration values used by the CLI. @@ -136,6 +140,9 @@ type AWSConfig struct { // description: | // Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage DeployCSIDriver *bool `yaml:"deployCSIDriver" validate:"required"` + // description: | + // Use the specified AWS Marketplace image offering. + UseMarketplaceImage *bool `yaml:"useMarketplaceImage" validate:"omitempty"` } // AzureConfig are Azure specific configuration values used by the CLI. @@ -181,8 +188,14 @@ type GCPConfig struct { // Path of service account key file. For required service account roles, see https://docs.edgeless.systems/constellation/getting-started/install#authorization ServiceAccountKeyPath string `yaml:"serviceAccountKeyPath" validate:"required"` // description: | + // GCP service account mail address. This is being attached to the VMs for authorization. + IAMServiceAccountVM string `yaml:"IAMServiceAccountVM"` + // description: | // Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage DeployCSIDriver *bool `yaml:"deployCSIDriver" validate:"required"` + // description: | + // Use the specified GCP Marketplace image offering. + UseMarketplaceImage *bool `yaml:"useMarketplaceImage" validate:"omitempty"` } // OpenStackConfig holds config information for OpenStack based Constellation deployments. @@ -191,39 +204,22 @@ type OpenStackConfig struct { // OpenStack cloud name to select from "clouds.yaml". Only required if config file for OpenStack is used. Fallback authentication uses environment variables. For details see: https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html. Cloud string `yaml:"cloud"` // description: | + // Path to OpenStack "clouds.yaml" file. Only required if automatic detection fails. + CloudsYAMLPath string `yaml:"cloudsYAMLPath"` + // description: | // Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html AvailabilityZone string `yaml:"availabilityZone" validate:"required"` // description: | // Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html FloatingIPPoolID string `yaml:"floatingIPPoolID" validate:"required"` // description: | - // AuthURL is the OpenStack Identity endpoint to use inside the cluster. - AuthURL string `yaml:"authURL" validate:"required"` - // description: | - // ProjectID is the ID of the project where a user resides. - ProjectID string `yaml:"projectID" validate:"required"` - // description: | - // ProjectName is the name of the project where a user resides. - ProjectName string `yaml:"projectName" validate:"required"` - // description: | - // UserDomainName is the name of the domain where a user resides. - UserDomainName string `yaml:"userDomainName" validate:"required"` - // description: | - // ProjectDomainName is the name of the domain where a project resides. - ProjectDomainName string `yaml:"projectDomainName" validate:"required"` + // STACKITProjectID is the ID of the STACKIT project where a user resides. + // Only used if cloud is "stackit". + STACKITProjectID string `yaml:"stackitProjectID"` // description: | // RegionName is the name of the region to use inside the cluster. RegionName string `yaml:"regionName" validate:"required"` // description: | - // Username to use inside the cluster. - Username string `yaml:"username" validate:"required"` - // description: | - // Password to use inside the cluster. You can instead use the environment variable "CONSTELL_OS_PASSWORD". - Password string `yaml:"password"` - // description: | - // If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack. - DirectDownload *bool `yaml:"directDownload" validate:"required"` - // description: | // Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol DeployYawolLoadBalancer *bool `yaml:"deployYawolLoadBalancer" validate:"required"` // description: | @@ -271,25 +267,31 @@ type QEMUConfig struct { type AttestationConfig struct { // description: | // AWS SEV-SNP attestation. - AWSSEVSNP *AWSSEVSNP `yaml:"awsSEVSNP,omitempty" validate:"omitempty,dive"` + AWSSEVSNP *AWSSEVSNP `yaml:"awsSEVSNP,omitempty" validate:"omitempty"` // description: | // AWS Nitro TPM attestation. - AWSNitroTPM *AWSNitroTPM `yaml:"awsNitroTPM,omitempty" validate:"omitempty,dive"` + AWSNitroTPM *AWSNitroTPM `yaml:"awsNitroTPM,omitempty" validate:"omitempty"` // description: | // Azure SEV-SNP attestation.\nFor details see: https://docs.edgeless.systems/constellation/architecture/attestation#cvm-verification - AzureSEVSNP *AzureSEVSNP `yaml:"azureSEVSNP,omitempty" validate:"omitempty,dive"` + AzureSEVSNP *AzureSEVSNP `yaml:"azureSEVSNP,omitempty" validate:"omitempty"` + // description: | + // Azure TDX attestation. + AzureTDX *AzureTDX `yaml:"azureTDX,omitempty" validate:"omitempty"` // description: | // Azure TPM attestation (Trusted Launch). - AzureTrustedLaunch *AzureTrustedLaunch `yaml:"azureTrustedLaunch,omitempty" validate:"omitempty,dive"` + AzureTrustedLaunch *AzureTrustedLaunch `yaml:"azureTrustedLaunch,omitempty" validate:"omitempty"` // description: | // GCP SEV-ES attestation. - GCPSEVES *GCPSEVES `yaml:"gcpSEVES,omitempty" validate:"omitempty,dive"` + GCPSEVES *GCPSEVES `yaml:"gcpSEVES,omitempty" validate:"omitempty"` + // description: | + // GCP SEV-SNP attestation. + GCPSEVSNP *GCPSEVSNP `yaml:"gcpSEVSNP,omitempty" validate:"omitempty"` // description: | // QEMU tdx attestation. - QEMUTDX *QEMUTDX `yaml:"qemuTDX,omitempty" validate:"omitempty,dive"` + QEMUTDX *QEMUTDX `yaml:"qemuTDX,omitempty" validate:"omitempty"` // description: | // QEMU vTPM attestation. - QEMUVTPM *QEMUVTPM `yaml:"qemuVTPM,omitempty" validate:"omitempty,dive"` + QEMUVTPM *QEMUVTPM `yaml:"qemuVTPM,omitempty" validate:"omitempty"` } // NodeGroup defines a group of nodes with the same role and configuration. @@ -326,12 +328,14 @@ func Default() *Config { KubernetesVersion: versions.Default, DebugCluster: toPtr(false), ServiceCIDR: "10.96.0.0/12", + Tags: cloudprovider.Tags{}, Provider: ProviderConfig{ AWS: &AWSConfig{ Region: "", IAMProfileControlPlane: "", IAMProfileWorkerNodes: "", DeployCSIDriver: toPtr(true), + UseMarketplaceImage: toPtr(false), }, Azure: &AzureConfig{ SubscriptionID: "", @@ -348,10 +352,11 @@ func Default() *Config { Region: "", Zone: "", ServiceAccountKeyPath: "", + IAMServiceAccountVM: "", DeployCSIDriver: toPtr(true), + UseMarketplaceImage: toPtr(false), }, OpenStack: &OpenStackConfig{ - DirectDownload: toPtr(true), DeployYawolLoadBalancer: toPtr(true), DeployCSIDriver: toPtr(true), }, @@ -393,8 +398,10 @@ func Default() *Config { AWSSEVSNP: DefaultForAWSSEVSNP(), AWSNitroTPM: &AWSNitroTPM{Measurements: measurements.DefaultsFor(cloudprovider.AWS, variant.AWSNitroTPM{})}, AzureSEVSNP: DefaultForAzureSEVSNP(), + AzureTDX: DefaultForAzureTDX(), AzureTrustedLaunch: &AzureTrustedLaunch{Measurements: measurements.DefaultsFor(cloudprovider.Azure, variant.AzureTrustedLaunch{})}, GCPSEVES: &GCPSEVES{Measurements: measurements.DefaultsFor(cloudprovider.GCP, variant.GCPSEVES{})}, + GCPSEVSNP: DefaultForGCPSEVSNP(), QEMUVTPM: &QEMUVTPM{Measurements: measurements.DefaultsFor(cloudprovider.QEMU, variant.QEMUVTPM{})}, }, } @@ -465,17 +472,27 @@ func New(fileHandler file.Handler, name string, fetcher attestationconfigapi.Fet return nil, err } + // Replace "latest" placeholders for attestation version numbers with the actual latest version numbers from config API if azure := c.Attestation.AzureSEVSNP; azure != nil { if err := azure.FetchAndSetLatestVersionNumbers(context.Background(), fetcher); err != nil { return c, err } } - + if azure := c.Attestation.AzureTDX; azure != nil { + if err := azure.FetchAndSetLatestVersionNumbers(context.Background(), fetcher); err != nil { + return c, err + } + } if aws := c.Attestation.AWSSEVSNP; aws != nil { if err := aws.FetchAndSetLatestVersionNumbers(context.Background(), fetcher); err != nil { return c, err } } + if gcp := c.Attestation.GCPSEVSNP; gcp != nil { + if err := gcp.FetchAndSetLatestVersionNumbers(context.Background(), fetcher); err != nil { + return c, err + } + } // Read secrets from env-vars. clientSecretValue := os.Getenv(constants.EnvVarAzureClientSecretValue) @@ -483,11 +500,6 @@ func New(fileHandler file.Handler, name string, fetcher attestationconfigapi.Fet fmt.Fprintf(os.Stderr, "WARNING: the environment variable %s is no longer used %s", constants.EnvVarAzureClientSecretValue, appRegistrationErrStr) } - openstackPassword := os.Getenv(constants.EnvVarOpenStackPassword) - if openstackPassword != "" && c.Provider.OpenStack != nil { - c.Provider.OpenStack.Password = openstackPassword - } - return c, c.Validate(force) } @@ -519,12 +531,18 @@ func (c *Config) UpdateMeasurements(newMeasurements measurements.M) { if c.Attestation.AzureSEVSNP != nil { c.Attestation.AzureSEVSNP.Measurements.CopyFrom(newMeasurements) } + if c.Attestation.AzureTDX != nil { + c.Attestation.AzureTDX.Measurements.CopyFrom(newMeasurements) + } if c.Attestation.AzureTrustedLaunch != nil { c.Attestation.AzureTrustedLaunch.Measurements.CopyFrom(newMeasurements) } if c.Attestation.GCPSEVES != nil { c.Attestation.GCPSEVES.Measurements.CopyFrom(newMeasurements) } + if c.Attestation.GCPSEVSNP != nil { + c.Attestation.GCPSEVSNP.Measurements.CopyFrom(newMeasurements) + } if c.Attestation.QEMUVTPM != nil { c.Attestation.QEMUVTPM.Measurements.CopyFrom(newMeasurements) } @@ -534,6 +552,7 @@ func (c *Config) UpdateMeasurements(newMeasurements measurements.M) { func (c *Config) RemoveProviderAndAttestationExcept(provider cloudprovider.Provider) { c.RemoveProviderExcept(provider) c.SetAttestation(variant.GetDefaultAttestation(provider)) + c.SetCSPNodeGroupDefaults(provider) } // RemoveProviderExcept removes all provider specific configurations, i.e., @@ -557,7 +576,6 @@ func (c *Config) RemoveProviderExcept(provider cloudprovider.Provider) { default: c.Provider = currentProviderConfigs } - c.setCSPNodeGroupDefaults(provider) } // SetAttestation sets the attestation config for the given attestation variant and removes all other attestation configs. @@ -565,16 +583,20 @@ func (c *Config) SetAttestation(attestation variant.Variant) { currentAttestationConfigs := c.Attestation c.Attestation = AttestationConfig{} switch attestation.(type) { - case variant.AzureSEVSNP: - c.Attestation = AttestationConfig{AzureSEVSNP: currentAttestationConfigs.AzureSEVSNP} case variant.AWSSEVSNP: c.Attestation = AttestationConfig{AWSSEVSNP: currentAttestationConfigs.AWSSEVSNP} case variant.AWSNitroTPM: c.Attestation = AttestationConfig{AWSNitroTPM: currentAttestationConfigs.AWSNitroTPM} + case variant.AzureSEVSNP: + c.Attestation = AttestationConfig{AzureSEVSNP: currentAttestationConfigs.AzureSEVSNP} + case variant.AzureTDX: + c.Attestation = AttestationConfig{AzureTDX: currentAttestationConfigs.AzureTDX} case variant.AzureTrustedLaunch: c.Attestation = AttestationConfig{AzureTrustedLaunch: currentAttestationConfigs.AzureTrustedLaunch} case variant.GCPSEVES: c.Attestation = AttestationConfig{GCPSEVES: currentAttestationConfigs.GCPSEVES} + case variant.GCPSEVSNP: + c.Attestation = AttestationConfig{GCPSEVSNP: currentAttestationConfigs.GCPSEVSNP} case variant.QEMUVTPM: c.Attestation = AttestationConfig{QEMUVTPM: currentAttestationConfigs.QEMUVTPM} } @@ -625,15 +647,16 @@ func (c *Config) GetProvider() cloudprovider.Provider { // GetAttestationConfig returns the configured attestation config. func (c *Config) GetAttestationConfig() AttestationCfg { if c.Attestation.AWSSEVSNP != nil { - return c.Attestation.AWSSEVSNP + return c.Attestation.AWSSEVSNP.getToMarshallLatestWithResolvedVersions() } if c.Attestation.AWSNitroTPM != nil { return c.Attestation.AWSNitroTPM } if c.Attestation.AzureSEVSNP != nil { - cp := *c.Attestation.AzureSEVSNP - cp.setWantLatestToFalse() - return &cp + return c.Attestation.AzureSEVSNP.getToMarshallLatestWithResolvedVersions() + } + if c.Attestation.AzureTDX != nil { + return c.Attestation.AzureTDX.getToMarshallLatestWithResolvedVersions() } if c.Attestation.AzureTrustedLaunch != nil { return c.Attestation.AzureTrustedLaunch @@ -641,6 +664,9 @@ func (c *Config) GetAttestationConfig() AttestationCfg { if c.Attestation.GCPSEVES != nil { return c.Attestation.GCPSEVES } + if c.Attestation.GCPSEVSNP != nil { + return c.Attestation.GCPSEVSNP + } if c.Attestation.QEMUVTPM != nil { return c.Attestation.QEMUVTPM } @@ -699,7 +725,10 @@ func (c *Config) DeployYawolLoadBalancer() bool { // UseMarketplaceImage returns whether a marketplace image should be used. func (c *Config) UseMarketplaceImage() bool { - return c.Provider.Azure != nil && c.Provider.Azure.UseMarketplaceImage != nil && *c.Provider.Azure.UseMarketplaceImage + return (c.Provider.Azure != nil && c.Provider.Azure.UseMarketplaceImage != nil && *c.Provider.Azure.UseMarketplaceImage) || + (c.Provider.GCP != nil && c.Provider.GCP.UseMarketplaceImage != nil && *c.Provider.GCP.UseMarketplaceImage) || + (c.Provider.AWS != nil && c.Provider.AWS.UseMarketplaceImage != nil && *c.Provider.AWS.UseMarketplaceImage) || + (c.Provider.OpenStack != nil && c.Provider.OpenStack.Cloud == "stackit") } // Validate checks the config values and returns validation errors. @@ -846,7 +875,7 @@ func (c *Config) Validate(force bool) error { // Because of this we can't print the offending field name in the error message, resulting in // suboptimal UX. Adding the field name to the struct validation of Semver would make it // impossible to use Semver for other fields. - if err := validateMicroserviceVersion(constants.BinaryVersion(), c.MicroserviceVersion); err != nil { + if err := ValidateMicroserviceVersion(constants.BinaryVersion(), c.MicroserviceVersion); err != nil { msg := "microserviceVersion: " + msgFromCompatibilityError(err, constants.BinaryVersion().String(), c.MicroserviceVersion.String()) return &ValidationError{validationErrMsgs: []string{msg}} } @@ -878,22 +907,22 @@ func (c *Config) Validate(force bool) error { // WithOpenStackProviderDefaults fills the default values for the specific OpenStack provider. // If the provider is not supported or not an OpenStack provider, the config is returned unchanged. -func (c *Config) WithOpenStackProviderDefaults(openStackProvider string) *Config { +func (c *Config) WithOpenStackProviderDefaults(csp cloudprovider.Provider, openStackProvider string) *Config { + if csp != cloudprovider.OpenStack { + return c + } + c.Attestation.QEMUVTPM = &QEMUVTPM{Measurements: measurements.DefaultsFor(cloudprovider.OpenStack, variant.QEMUVTPM{})} switch openStackProvider { case "stackit": c.Provider.OpenStack.Cloud = "stackit" c.Provider.OpenStack.FloatingIPPoolID = "970ace5c-458f-484a-a660-0903bcfd91ad" - c.Provider.OpenStack.AuthURL = "https://keystone.api.iaas.eu01.stackit.cloud/v3" - c.Provider.OpenStack.UserDomainName = "portal_mvp" - c.Provider.OpenStack.ProjectDomainName = "portal_mvp" c.Provider.OpenStack.RegionName = "RegionOne" c.Provider.OpenStack.DeployYawolLoadBalancer = toPtr(true) - c.Provider.OpenStack.YawolImageID = "43d9bede-1e7a-4ca7-82c5-0a5c72388619" + c.Provider.OpenStack.YawolImageID = "bcd6c13e-75d1-4c3f-bf0f-8f83580cc1be" c.Provider.OpenStack.YawolFlavorID = "3b11b27e-6c73-470d-b595-1d85b95a8cdf" c.Provider.OpenStack.DeployCSIDriver = toPtr(true) - c.Provider.OpenStack.DirectDownload = toPtr(true) for groupName, group := range c.NodeGroups { - group.InstanceType = "2715eabe-3ffc-4c36-b02a-efa8c141a96a" + group.InstanceType = "m1a.4cd" group.StateDiskType = "storage_premium_perf6" c.NodeGroups[groupName] = group } @@ -902,7 +931,8 @@ func (c *Config) WithOpenStackProviderDefaults(openStackProvider string) *Config return c } -func (c *Config) setCSPNodeGroupDefaults(csp cloudprovider.Provider) { +// SetCSPNodeGroupDefaults sets the default values for the node groups based on the configured CSP. +func (c *Config) SetCSPNodeGroupDefaults(csp cloudprovider.Provider) { var instanceType, stateDiskType, zone string switch csp { case cloudprovider.AWS: @@ -910,14 +940,19 @@ func (c *Config) setCSPNodeGroupDefaults(csp cloudprovider.Provider) { stateDiskType = "gp3" zone = c.Provider.AWS.Zone case cloudprovider.Azure: - instanceType = "Standard_DC4as_v5" + // Check attestation variant, and use different default instance type if we have TDX + if c.GetAttestationConfig().GetVariant().Equal(variant.AzureTDX{}) { + instanceType = "Standard_DC4es_v5" + } else { + instanceType = "Standard_DC4as_v5" + } stateDiskType = "Premium_LRS" case cloudprovider.GCP: instanceType = "n2d-standard-4" stateDiskType = "pd-ssd" zone = c.Provider.GCP.Zone case cloudprovider.QEMU, cloudprovider.OpenStack: - // empty. There are now defaults for this CSP + // empty. There are no defaults for this CSP } for groupName, group := range c.NodeGroups { @@ -959,32 +994,29 @@ type GCPSEVES struct { Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` } -// GetVariant returns gcp-sev-es as the variant. -func (GCPSEVES) GetVariant() variant.Variant { - return variant.GCPSEVES{} -} - -// GetMeasurements returns the measurements used for attestation. -func (c GCPSEVES) GetMeasurements() measurements.M { - return c.Measurements -} - -// SetMeasurements updates a config's measurements using the given measurements. -func (c *GCPSEVES) SetMeasurements(m measurements.M) { - c.Measurements = m -} - -// EqualTo returns true if the config is equal to the given config. -func (c GCPSEVES) EqualTo(other AttestationCfg) (bool, error) { - otherCfg, ok := other.(*GCPSEVES) - if !ok { - return false, fmt.Errorf("cannot compare %T with %T", c, other) - } - return c.Measurements.EqualTo(otherCfg.Measurements), nil -} - -func toPtr[T any](v T) *T { - return &v +// GCPSEVSNP is the configuration for GCP SEV-SNP attestation. +type GCPSEVSNP struct { + // description: | + // Expected TPM measurements. + Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` + // description: | + // Lowest acceptable bootloader version. + BootloaderVersion AttestationVersion[uint8] `json:"bootloaderVersion" yaml:"bootloaderVersion"` + // description: | + // Lowest acceptable TEE version. + TEEVersion AttestationVersion[uint8] `json:"teeVersion" yaml:"teeVersion"` + // description: | + // Lowest acceptable SEV-SNP version. + SNPVersion AttestationVersion[uint8] `json:"snpVersion" yaml:"snpVersion"` + // description: | + // Lowest acceptable microcode version. + MicrocodeVersion AttestationVersion[uint8] `json:"microcodeVersion" yaml:"microcodeVersion"` + // description: | + // AMD Root Key certificate used to verify the SEV-SNP certificate chain. + AMDRootKey Certificate `json:"amdRootKey" yaml:"amdRootKey"` + // description: | + // AMD Signing Key certificate used to verify the SEV-SNP VCEK / VLEK certificate. + AMDSigningKey Certificate `json:"amdSigningKey,omitempty" yaml:"amdSigningKey,omitempty"` } // QEMUVTPM is the configuration for QEMU vTPM attestation. @@ -1056,16 +1088,16 @@ type AWSSEVSNP struct { Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` // description: | // Lowest acceptable bootloader version. - BootloaderVersion AttestationVersion `json:"bootloaderVersion" yaml:"bootloaderVersion"` + BootloaderVersion AttestationVersion[uint8] `json:"bootloaderVersion" yaml:"bootloaderVersion"` // description: | // Lowest acceptable TEE version. - TEEVersion AttestationVersion `json:"teeVersion" yaml:"teeVersion"` + TEEVersion AttestationVersion[uint8] `json:"teeVersion" yaml:"teeVersion"` // description: | // Lowest acceptable SEV-SNP version. - SNPVersion AttestationVersion `json:"snpVersion" yaml:"snpVersion"` + SNPVersion AttestationVersion[uint8] `json:"snpVersion" yaml:"snpVersion"` // description: | // Lowest acceptable microcode version. - MicrocodeVersion AttestationVersion `json:"microcodeVersion" yaml:"microcodeVersion"` + MicrocodeVersion AttestationVersion[uint8] `json:"microcodeVersion" yaml:"microcodeVersion"` // description: | // AMD Root Key certificate used to verify the SEV-SNP certificate chain. AMDRootKey Certificate `json:"amdRootKey" yaml:"amdRootKey"` @@ -1088,16 +1120,16 @@ type AzureSEVSNP struct { Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` // description: | // Lowest acceptable bootloader version. - BootloaderVersion AttestationVersion `json:"bootloaderVersion" yaml:"bootloaderVersion"` + BootloaderVersion AttestationVersion[uint8] `json:"bootloaderVersion" yaml:"bootloaderVersion"` // description: | // Lowest acceptable TEE version. - TEEVersion AttestationVersion `json:"teeVersion" yaml:"teeVersion"` + TEEVersion AttestationVersion[uint8] `json:"teeVersion" yaml:"teeVersion"` // description: | // Lowest acceptable SEV-SNP version. - SNPVersion AttestationVersion `json:"snpVersion" yaml:"snpVersion"` + SNPVersion AttestationVersion[uint8] `json:"snpVersion" yaml:"snpVersion"` // description: | // Lowest acceptable microcode version. - MicrocodeVersion AttestationVersion `json:"microcodeVersion" yaml:"microcodeVersion"` + MicrocodeVersion AttestationVersion[uint8] `json:"microcodeVersion" yaml:"microcodeVersion"` // description: | // Configuration for validating the firmware signature. FirmwareSignerConfig SNPFirmwareSignerConfig `json:"firmwareSignerConfig" yaml:"firmwareSignerConfig"` @@ -1106,15 +1138,7 @@ type AzureSEVSNP struct { AMDRootKey Certificate `json:"amdRootKey" yaml:"amdRootKey"` // description: | // AMD Signing Key certificate used to verify the SEV-SNP VCEK / VLEK certificate. - AMDSigningKey Certificate `json:"amdSigningKey,omitempty" yaml:"amdSigningKey,omitempty" validate:"len=0"` -} - -// setWantLatestToFalse sets the WantLatest field to false for all versions in order to unmarshal the numerical versions instead of the string "latest". -func (c *AzureSEVSNP) setWantLatestToFalse() { - c.BootloaderVersion.WantLatest = false - c.TEEVersion.WantLatest = false - c.SNPVersion.WantLatest = false - c.MicrocodeVersion.WantLatest = false + AMDSigningKey Certificate `json:"amdSigningKey,omitempty" yaml:"amdSigningKey,omitempty"` } // AzureTrustedLaunch is the configuration for Azure Trusted Launch attestation. @@ -1123,3 +1147,41 @@ type AzureTrustedLaunch struct { // Expected TPM measurements. Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` } + +// AzureTDX is the configuration for Azure TDX attestation. +type AzureTDX struct { + // description: | + // Expected TPM measurements. + Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` + // description: | + // Minimum required QE security version number (SVN). + QESVN AttestationVersion[uint16] `json:"qeSVN" yaml:"qeSVN"` + // description: | + // Minimum required PCE security version number (SVN). + PCESVN AttestationVersion[uint16] `json:"pceSVN" yaml:"pceSVN"` + // description: | + // Component-wise minimum required 16 byte hex-encoded TEE_TCB security version number (SVN). + TEETCBSVN AttestationVersion[encoding.HexBytes] `json:"teeTCBSVN" yaml:"teeTCBSVN"` + // description: | + // Expected 16 byte hex-encoded QE_VENDOR_ID field. + QEVendorID AttestationVersion[encoding.HexBytes] `json:"qeVendorID" yaml:"qeVendorID"` + // description: | + // Expected 48 byte hex-encoded MR_SEAM value. + MRSeam encoding.HexBytes `json:"mrSeam,omitempty" yaml:"mrSeam,omitempty"` + // description: | + // Expected 8 byte hex-encoded eXtended Features Available Mask (XFAM) field. Defaults to the latest available XFAM on Azure VMs. Unset to disable validation. + XFAM AttestationVersion[encoding.HexBytes] `json:"xfam" yaml:"xfam"` + // description: | + // Intel Root Key certificate used to verify the TDX certificate chain. + IntelRootKey Certificate `json:"intelRootKey" yaml:"intelRootKey"` +} + +func toPtr[T any](v T) *T { + return &v +} + +// svnResolveMarshaller is used to marshall "latest" security version numbers with resolved versions. +type svnResolveMarshaller interface { + // getToMarshallLatestWithResolvedVersions brings the attestation config into a state where marshalling uses the numerical version numbers for "latest" versions. + getToMarshallLatestWithResolvedVersions() AttestationCfg +} diff --git a/internal/config/config_doc.go b/internal/config/config_doc.go index dab022b47..b87db6b86 100644 --- a/internal/config/config_doc.go +++ b/internal/config/config_doc.go @@ -23,19 +23,21 @@ var ( UnsupportedAppRegistrationErrorDoc encoder.Doc SNPFirmwareSignerConfigDoc encoder.Doc GCPSEVESDoc encoder.Doc + GCPSEVSNPDoc encoder.Doc QEMUVTPMDoc encoder.Doc QEMUTDXDoc encoder.Doc AWSSEVSNPDoc encoder.Doc AWSNitroTPMDoc encoder.Doc AzureSEVSNPDoc encoder.Doc AzureTrustedLaunchDoc encoder.Doc + AzureTDXDoc encoder.Doc ) func init() { ConfigDoc.Type = "Config" ConfigDoc.Comments[encoder.LineComment] = "Config defines configuration used by CLI." ConfigDoc.Description = "Config defines configuration used by CLI." - ConfigDoc.Fields = make([]encoder.Doc, 12) + ConfigDoc.Fields = make([]encoder.Doc, 13) ConfigDoc.Fields[0].Name = "version" ConfigDoc.Fields[0].Type = "string" ConfigDoc.Fields[0].Note = "" @@ -81,21 +83,26 @@ func init() { ConfigDoc.Fields[8].Note = "" ConfigDoc.Fields[8].Description = "The Kubernetes Service CIDR to be used for the cluster. This value will only be used during the first initialization of the Constellation." ConfigDoc.Fields[8].Comments[encoder.LineComment] = "The Kubernetes Service CIDR to be used for the cluster. This value will only be used during the first initialization of the Constellation." - ConfigDoc.Fields[9].Name = "provider" - ConfigDoc.Fields[9].Type = "ProviderConfig" + ConfigDoc.Fields[9].Name = "tags" + ConfigDoc.Fields[9].Type = "Tags" ConfigDoc.Fields[9].Note = "" - ConfigDoc.Fields[9].Description = "Supported cloud providers and their specific configurations." - ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Supported cloud providers and their specific configurations." - ConfigDoc.Fields[10].Name = "nodeGroups" - ConfigDoc.Fields[10].Type = "map[string]NodeGroup" + ConfigDoc.Fields[9].Description = "Additional tags that are applied to created resources." + ConfigDoc.Fields[9].Comments[encoder.LineComment] = "Additional tags that are applied to created resources." + ConfigDoc.Fields[10].Name = "provider" + ConfigDoc.Fields[10].Type = "ProviderConfig" ConfigDoc.Fields[10].Note = "" - ConfigDoc.Fields[10].Description = "Node groups to be created in the cluster." - ConfigDoc.Fields[10].Comments[encoder.LineComment] = "Node groups to be created in the cluster." - ConfigDoc.Fields[11].Name = "attestation" - ConfigDoc.Fields[11].Type = "AttestationConfig" + ConfigDoc.Fields[10].Description = "Supported cloud providers and their specific configurations." + ConfigDoc.Fields[10].Comments[encoder.LineComment] = "Supported cloud providers and their specific configurations." + ConfigDoc.Fields[11].Name = "nodeGroups" + ConfigDoc.Fields[11].Type = "map[string]NodeGroup" ConfigDoc.Fields[11].Note = "" - ConfigDoc.Fields[11].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" - ConfigDoc.Fields[11].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" + ConfigDoc.Fields[11].Description = "Node groups to be created in the cluster." + ConfigDoc.Fields[11].Comments[encoder.LineComment] = "Node groups to be created in the cluster." + ConfigDoc.Fields[12].Name = "attestation" + ConfigDoc.Fields[12].Type = "AttestationConfig" + ConfigDoc.Fields[12].Note = "" + ConfigDoc.Fields[12].Description = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" + ConfigDoc.Fields[12].Comments[encoder.LineComment] = "Configuration for attestation validation. This configuration provides sensible defaults for the Constellation version it was created for.\nSee the docs for an overview on attestation: https://docs.edgeless.systems/constellation/architecture/attestation" ProviderConfigDoc.Type = "ProviderConfig" ProviderConfigDoc.Comments[encoder.LineComment] = "ProviderConfig are cloud-provider specific configuration values used by the CLI." @@ -142,7 +149,7 @@ func init() { FieldName: "aws", }, } - AWSConfigDoc.Fields = make([]encoder.Doc, 5) + AWSConfigDoc.Fields = make([]encoder.Doc, 6) AWSConfigDoc.Fields[0].Name = "region" AWSConfigDoc.Fields[0].Type = "string" AWSConfigDoc.Fields[0].Note = "" @@ -168,6 +175,11 @@ func init() { AWSConfigDoc.Fields[4].Note = "" AWSConfigDoc.Fields[4].Description = "Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" AWSConfigDoc.Fields[4].Comments[encoder.LineComment] = "Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + AWSConfigDoc.Fields[5].Name = "useMarketplaceImage" + AWSConfigDoc.Fields[5].Type = "bool" + AWSConfigDoc.Fields[5].Note = "" + AWSConfigDoc.Fields[5].Description = "Use the specified AWS Marketplace image offering." + AWSConfigDoc.Fields[5].Comments[encoder.LineComment] = "Use the specified AWS Marketplace image offering." AzureConfigDoc.Type = "AzureConfig" AzureConfigDoc.Comments[encoder.LineComment] = "AzureConfig are Azure specific configuration values used by the CLI." @@ -229,7 +241,7 @@ func init() { FieldName: "gcp", }, } - GCPConfigDoc.Fields = make([]encoder.Doc, 5) + GCPConfigDoc.Fields = make([]encoder.Doc, 7) GCPConfigDoc.Fields[0].Name = "project" GCPConfigDoc.Fields[0].Type = "string" GCPConfigDoc.Fields[0].Note = "" @@ -250,11 +262,21 @@ func init() { GCPConfigDoc.Fields[3].Note = "" GCPConfigDoc.Fields[3].Description = "Path of service account key file. For required service account roles, see https://docs.edgeless.systems/constellation/getting-started/install#authorization" GCPConfigDoc.Fields[3].Comments[encoder.LineComment] = "Path of service account key file. For required service account roles, see https://docs.edgeless.systems/constellation/getting-started/install#authorization" - GCPConfigDoc.Fields[4].Name = "deployCSIDriver" - GCPConfigDoc.Fields[4].Type = "bool" + GCPConfigDoc.Fields[4].Name = "IAMServiceAccountVM" + GCPConfigDoc.Fields[4].Type = "string" GCPConfigDoc.Fields[4].Note = "" - GCPConfigDoc.Fields[4].Description = "Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" - GCPConfigDoc.Fields[4].Comments[encoder.LineComment] = "Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + GCPConfigDoc.Fields[4].Description = "GCP service account mail address. This is being attached to the VMs for authorization." + GCPConfigDoc.Fields[4].Comments[encoder.LineComment] = "GCP service account mail address. This is being attached to the VMs for authorization." + GCPConfigDoc.Fields[5].Name = "deployCSIDriver" + GCPConfigDoc.Fields[5].Type = "bool" + GCPConfigDoc.Fields[5].Note = "" + GCPConfigDoc.Fields[5].Description = "Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + GCPConfigDoc.Fields[5].Comments[encoder.LineComment] = "Deploy Persistent Disk CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + GCPConfigDoc.Fields[6].Name = "useMarketplaceImage" + GCPConfigDoc.Fields[6].Type = "bool" + GCPConfigDoc.Fields[6].Note = "" + GCPConfigDoc.Fields[6].Description = "Use the specified GCP Marketplace image offering." + GCPConfigDoc.Fields[6].Comments[encoder.LineComment] = "Use the specified GCP Marketplace image offering." OpenStackConfigDoc.Type = "OpenStackConfig" OpenStackConfigDoc.Comments[encoder.LineComment] = "OpenStackConfig holds config information for OpenStack based Constellation deployments." @@ -265,87 +287,57 @@ func init() { FieldName: "openstack", }, } - OpenStackConfigDoc.Fields = make([]encoder.Doc, 16) + OpenStackConfigDoc.Fields = make([]encoder.Doc, 10) OpenStackConfigDoc.Fields[0].Name = "cloud" OpenStackConfigDoc.Fields[0].Type = "string" OpenStackConfigDoc.Fields[0].Note = "" OpenStackConfigDoc.Fields[0].Description = "OpenStack cloud name to select from \"clouds.yaml\". Only required if config file for OpenStack is used. Fallback authentication uses environment variables. For details see: https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html." OpenStackConfigDoc.Fields[0].Comments[encoder.LineComment] = "OpenStack cloud name to select from \"clouds.yaml\". Only required if config file for OpenStack is used. Fallback authentication uses environment variables. For details see: https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html." - OpenStackConfigDoc.Fields[1].Name = "availabilityZone" + OpenStackConfigDoc.Fields[1].Name = "cloudsYAMLPath" OpenStackConfigDoc.Fields[1].Type = "string" OpenStackConfigDoc.Fields[1].Note = "" - OpenStackConfigDoc.Fields[1].Description = "Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html" - OpenStackConfigDoc.Fields[1].Comments[encoder.LineComment] = "Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html" - OpenStackConfigDoc.Fields[2].Name = "floatingIPPoolID" + OpenStackConfigDoc.Fields[1].Description = "Path to OpenStack \"clouds.yaml\" file. Only required if automatic detection fails." + OpenStackConfigDoc.Fields[1].Comments[encoder.LineComment] = "Path to OpenStack \"clouds.yaml\" file. Only required if automatic detection fails." + OpenStackConfigDoc.Fields[2].Name = "availabilityZone" OpenStackConfigDoc.Fields[2].Type = "string" OpenStackConfigDoc.Fields[2].Note = "" - OpenStackConfigDoc.Fields[2].Description = "Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html" - OpenStackConfigDoc.Fields[2].Comments[encoder.LineComment] = "Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html" - OpenStackConfigDoc.Fields[3].Name = "authURL" + OpenStackConfigDoc.Fields[2].Description = "Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html" + OpenStackConfigDoc.Fields[2].Comments[encoder.LineComment] = "Availability zone to place the VMs in. For details see: https://docs.openstack.org/nova/latest/admin/availability-zones.html" + OpenStackConfigDoc.Fields[3].Name = "floatingIPPoolID" OpenStackConfigDoc.Fields[3].Type = "string" OpenStackConfigDoc.Fields[3].Note = "" - OpenStackConfigDoc.Fields[3].Description = "description: |\nAuthURL is the OpenStack Identity endpoint to use inside the cluster.\n" - OpenStackConfigDoc.Fields[3].Comments[encoder.LineComment] = "description: |" - OpenStackConfigDoc.Fields[4].Name = "projectID" + OpenStackConfigDoc.Fields[3].Description = "Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html" + OpenStackConfigDoc.Fields[3].Comments[encoder.LineComment] = "Floating IP pool to use for the VMs. For details see: https://docs.openstack.org/ocata/user-guide/cli-manage-ip-addresses.html" + OpenStackConfigDoc.Fields[4].Name = "stackitProjectID" OpenStackConfigDoc.Fields[4].Type = "string" OpenStackConfigDoc.Fields[4].Note = "" - OpenStackConfigDoc.Fields[4].Description = "ProjectID is the ID of the project where a user resides." - OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "ProjectID is the ID of the project where a user resides." - OpenStackConfigDoc.Fields[5].Name = "projectName" + OpenStackConfigDoc.Fields[4].Description = "STACKITProjectID is the ID of the STACKIT project where a user resides.\nOnly used if cloud is \"stackit\"." + OpenStackConfigDoc.Fields[4].Comments[encoder.LineComment] = "STACKITProjectID is the ID of the STACKIT project where a user resides." + OpenStackConfigDoc.Fields[5].Name = "regionName" OpenStackConfigDoc.Fields[5].Type = "string" OpenStackConfigDoc.Fields[5].Note = "" - OpenStackConfigDoc.Fields[5].Description = "ProjectName is the name of the project where a user resides." - OpenStackConfigDoc.Fields[5].Comments[encoder.LineComment] = "ProjectName is the name of the project where a user resides." - OpenStackConfigDoc.Fields[6].Name = "userDomainName" - OpenStackConfigDoc.Fields[6].Type = "string" + OpenStackConfigDoc.Fields[5].Description = "description: |\nRegionName is the name of the region to use inside the cluster.\n" + OpenStackConfigDoc.Fields[5].Comments[encoder.LineComment] = "description: |" + OpenStackConfigDoc.Fields[6].Name = "deployYawolLoadBalancer" + OpenStackConfigDoc.Fields[6].Type = "bool" OpenStackConfigDoc.Fields[6].Note = "" - OpenStackConfigDoc.Fields[6].Description = "UserDomainName is the name of the domain where a user resides." - OpenStackConfigDoc.Fields[6].Comments[encoder.LineComment] = "UserDomainName is the name of the domain where a user resides." - OpenStackConfigDoc.Fields[7].Name = "projectDomainName" + OpenStackConfigDoc.Fields[6].Description = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[6].Comments[encoder.LineComment] = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[7].Name = "yawolImageID" OpenStackConfigDoc.Fields[7].Type = "string" OpenStackConfigDoc.Fields[7].Note = "" - OpenStackConfigDoc.Fields[7].Description = "ProjectDomainName is the name of the domain where a project resides." - OpenStackConfigDoc.Fields[7].Comments[encoder.LineComment] = "ProjectDomainName is the name of the domain where a project resides." - OpenStackConfigDoc.Fields[8].Name = "regionName" + OpenStackConfigDoc.Fields[7].Description = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[7].Comments[encoder.LineComment] = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[8].Name = "yawolFlavorID" OpenStackConfigDoc.Fields[8].Type = "string" OpenStackConfigDoc.Fields[8].Note = "" - OpenStackConfigDoc.Fields[8].Description = "description: |\nRegionName is the name of the region to use inside the cluster.\n" - OpenStackConfigDoc.Fields[8].Comments[encoder.LineComment] = "description: |" - OpenStackConfigDoc.Fields[9].Name = "username" - OpenStackConfigDoc.Fields[9].Type = "string" + OpenStackConfigDoc.Fields[8].Description = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[8].Comments[encoder.LineComment] = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" + OpenStackConfigDoc.Fields[9].Name = "deployCSIDriver" + OpenStackConfigDoc.Fields[9].Type = "bool" OpenStackConfigDoc.Fields[9].Note = "" - OpenStackConfigDoc.Fields[9].Description = "Username to use inside the cluster." - OpenStackConfigDoc.Fields[9].Comments[encoder.LineComment] = "Username to use inside the cluster." - OpenStackConfigDoc.Fields[10].Name = "password" - OpenStackConfigDoc.Fields[10].Type = "string" - OpenStackConfigDoc.Fields[10].Note = "" - OpenStackConfigDoc.Fields[10].Description = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"." - OpenStackConfigDoc.Fields[10].Comments[encoder.LineComment] = "Password to use inside the cluster. You can instead use the environment variable \"CONSTELL_OS_PASSWORD\"." - OpenStackConfigDoc.Fields[11].Name = "directDownload" - OpenStackConfigDoc.Fields[11].Type = "bool" - OpenStackConfigDoc.Fields[11].Note = "" - OpenStackConfigDoc.Fields[11].Description = "If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack." - OpenStackConfigDoc.Fields[11].Comments[encoder.LineComment] = "If enabled, downloads OS image directly from source URL to OpenStack. Otherwise, downloads image to local machine and uploads to OpenStack." - OpenStackConfigDoc.Fields[12].Name = "deployYawolLoadBalancer" - OpenStackConfigDoc.Fields[12].Type = "bool" - OpenStackConfigDoc.Fields[12].Note = "" - OpenStackConfigDoc.Fields[12].Description = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[12].Comments[encoder.LineComment] = "Deploy Yawol loadbalancer. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[13].Name = "yawolImageID" - OpenStackConfigDoc.Fields[13].Type = "string" - OpenStackConfigDoc.Fields[13].Note = "" - OpenStackConfigDoc.Fields[13].Description = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[13].Comments[encoder.LineComment] = "OpenStack OS image used by the yawollet. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[14].Name = "yawolFlavorID" - OpenStackConfigDoc.Fields[14].Type = "string" - OpenStackConfigDoc.Fields[14].Note = "" - OpenStackConfigDoc.Fields[14].Description = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[14].Comments[encoder.LineComment] = "OpenStack flavor id used for yawollets. For details see: https://github.com/stackitcloud/yawol" - OpenStackConfigDoc.Fields[15].Name = "deployCSIDriver" - OpenStackConfigDoc.Fields[15].Type = "bool" - OpenStackConfigDoc.Fields[15].Note = "" - OpenStackConfigDoc.Fields[15].Description = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" - OpenStackConfigDoc.Fields[15].Comments[encoder.LineComment] = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + OpenStackConfigDoc.Fields[9].Description = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" + OpenStackConfigDoc.Fields[9].Comments[encoder.LineComment] = "Deploy Cinder CSI driver with on-node encryption. For details see: https://docs.edgeless.systems/constellation/architecture/encrypted-storage" QEMUConfigDoc.Type = "QEMUConfig" QEMUConfigDoc.Comments[encoder.LineComment] = "QEMUConfig holds config information for QEMU based Constellation deployments." @@ -407,7 +399,7 @@ func init() { FieldName: "attestation", }, } - AttestationConfigDoc.Fields = make([]encoder.Doc, 7) + AttestationConfigDoc.Fields = make([]encoder.Doc, 9) AttestationConfigDoc.Fields[0].Name = "awsSEVSNP" AttestationConfigDoc.Fields[0].Type = "AWSSEVSNP" AttestationConfigDoc.Fields[0].Note = "" @@ -423,26 +415,36 @@ func init() { AttestationConfigDoc.Fields[2].Note = "" AttestationConfigDoc.Fields[2].Description = "Azure SEV-SNP attestation.\nFor details see: https://docs.edgeless.systems/constellation/architecture/attestation#cvm-verification" AttestationConfigDoc.Fields[2].Comments[encoder.LineComment] = "Azure SEV-SNP attestation.\nFor details see: https://docs.edgeless.systems/constellation/architecture/attestation#cvm-verification" - AttestationConfigDoc.Fields[3].Name = "azureTrustedLaunch" - AttestationConfigDoc.Fields[3].Type = "AzureTrustedLaunch" + AttestationConfigDoc.Fields[3].Name = "azureTDX" + AttestationConfigDoc.Fields[3].Type = "AzureTDX" AttestationConfigDoc.Fields[3].Note = "" - AttestationConfigDoc.Fields[3].Description = "Azure TPM attestation (Trusted Launch)." - AttestationConfigDoc.Fields[3].Comments[encoder.LineComment] = "Azure TPM attestation (Trusted Launch)." - AttestationConfigDoc.Fields[4].Name = "gcpSEVES" - AttestationConfigDoc.Fields[4].Type = "GCPSEVES" + AttestationConfigDoc.Fields[3].Description = "Azure TDX attestation." + AttestationConfigDoc.Fields[3].Comments[encoder.LineComment] = "Azure TDX attestation." + AttestationConfigDoc.Fields[4].Name = "azureTrustedLaunch" + AttestationConfigDoc.Fields[4].Type = "AzureTrustedLaunch" AttestationConfigDoc.Fields[4].Note = "" - AttestationConfigDoc.Fields[4].Description = "GCP SEV-ES attestation." - AttestationConfigDoc.Fields[4].Comments[encoder.LineComment] = "GCP SEV-ES attestation." - AttestationConfigDoc.Fields[5].Name = "qemuTDX" - AttestationConfigDoc.Fields[5].Type = "QEMUTDX" + AttestationConfigDoc.Fields[4].Description = "Azure TPM attestation (Trusted Launch)." + AttestationConfigDoc.Fields[4].Comments[encoder.LineComment] = "Azure TPM attestation (Trusted Launch)." + AttestationConfigDoc.Fields[5].Name = "gcpSEVES" + AttestationConfigDoc.Fields[5].Type = "GCPSEVES" AttestationConfigDoc.Fields[5].Note = "" - AttestationConfigDoc.Fields[5].Description = "QEMU tdx attestation." - AttestationConfigDoc.Fields[5].Comments[encoder.LineComment] = "QEMU tdx attestation." - AttestationConfigDoc.Fields[6].Name = "qemuVTPM" - AttestationConfigDoc.Fields[6].Type = "QEMUVTPM" + AttestationConfigDoc.Fields[5].Description = "GCP SEV-ES attestation." + AttestationConfigDoc.Fields[5].Comments[encoder.LineComment] = "GCP SEV-ES attestation." + AttestationConfigDoc.Fields[6].Name = "gcpSEVSNP" + AttestationConfigDoc.Fields[6].Type = "GCPSEVSNP" AttestationConfigDoc.Fields[6].Note = "" - AttestationConfigDoc.Fields[6].Description = "QEMU vTPM attestation." - AttestationConfigDoc.Fields[6].Comments[encoder.LineComment] = "QEMU vTPM attestation." + AttestationConfigDoc.Fields[6].Description = "GCP SEV-SNP attestation." + AttestationConfigDoc.Fields[6].Comments[encoder.LineComment] = "GCP SEV-SNP attestation." + AttestationConfigDoc.Fields[7].Name = "qemuTDX" + AttestationConfigDoc.Fields[7].Type = "QEMUTDX" + AttestationConfigDoc.Fields[7].Note = "" + AttestationConfigDoc.Fields[7].Description = "QEMU tdx attestation." + AttestationConfigDoc.Fields[7].Comments[encoder.LineComment] = "QEMU tdx attestation." + AttestationConfigDoc.Fields[8].Name = "qemuVTPM" + AttestationConfigDoc.Fields[8].Type = "QEMUVTPM" + AttestationConfigDoc.Fields[8].Note = "" + AttestationConfigDoc.Fields[8].Description = "QEMU vTPM attestation." + AttestationConfigDoc.Fields[8].Comments[encoder.LineComment] = "QEMU vTPM attestation." NodeGroupDoc.Type = "NodeGroup" NodeGroupDoc.Comments[encoder.LineComment] = "NodeGroup defines a group of nodes with the same role and configuration." @@ -532,6 +534,52 @@ func init() { GCPSEVESDoc.Fields[0].Description = "Expected TPM measurements." GCPSEVESDoc.Fields[0].Comments[encoder.LineComment] = "Expected TPM measurements." + GCPSEVSNPDoc.Type = "GCPSEVSNP" + GCPSEVSNPDoc.Comments[encoder.LineComment] = "GCPSEVSNP is the configuration for GCP SEV-SNP attestation." + GCPSEVSNPDoc.Description = "GCPSEVSNP is the configuration for GCP SEV-SNP attestation." + GCPSEVSNPDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "AttestationConfig", + FieldName: "gcpSEVSNP", + }, + } + GCPSEVSNPDoc.Fields = make([]encoder.Doc, 7) + GCPSEVSNPDoc.Fields[0].Name = "measurements" + GCPSEVSNPDoc.Fields[0].Type = "M" + GCPSEVSNPDoc.Fields[0].Note = "" + GCPSEVSNPDoc.Fields[0].Description = "Expected TPM measurements." + GCPSEVSNPDoc.Fields[0].Comments[encoder.LineComment] = "Expected TPM measurements." + GCPSEVSNPDoc.Fields[1].Name = "bootloaderVersion" + GCPSEVSNPDoc.Fields[1].Type = "" + GCPSEVSNPDoc.Fields[1].Note = "" + GCPSEVSNPDoc.Fields[1].Description = "Lowest acceptable bootloader version." + GCPSEVSNPDoc.Fields[1].Comments[encoder.LineComment] = "Lowest acceptable bootloader version." + GCPSEVSNPDoc.Fields[2].Name = "teeVersion" + GCPSEVSNPDoc.Fields[2].Type = "" + GCPSEVSNPDoc.Fields[2].Note = "" + GCPSEVSNPDoc.Fields[2].Description = "Lowest acceptable TEE version." + GCPSEVSNPDoc.Fields[2].Comments[encoder.LineComment] = "Lowest acceptable TEE version." + GCPSEVSNPDoc.Fields[3].Name = "snpVersion" + GCPSEVSNPDoc.Fields[3].Type = "" + GCPSEVSNPDoc.Fields[3].Note = "" + GCPSEVSNPDoc.Fields[3].Description = "Lowest acceptable SEV-SNP version." + GCPSEVSNPDoc.Fields[3].Comments[encoder.LineComment] = "Lowest acceptable SEV-SNP version." + GCPSEVSNPDoc.Fields[4].Name = "microcodeVersion" + GCPSEVSNPDoc.Fields[4].Type = "" + GCPSEVSNPDoc.Fields[4].Note = "" + GCPSEVSNPDoc.Fields[4].Description = "Lowest acceptable microcode version." + GCPSEVSNPDoc.Fields[4].Comments[encoder.LineComment] = "Lowest acceptable microcode version." + GCPSEVSNPDoc.Fields[5].Name = "amdRootKey" + GCPSEVSNPDoc.Fields[5].Type = "Certificate" + GCPSEVSNPDoc.Fields[5].Note = "" + GCPSEVSNPDoc.Fields[5].Description = "AMD Root Key certificate used to verify the SEV-SNP certificate chain." + GCPSEVSNPDoc.Fields[5].Comments[encoder.LineComment] = "AMD Root Key certificate used to verify the SEV-SNP certificate chain." + GCPSEVSNPDoc.Fields[6].Name = "amdSigningKey" + GCPSEVSNPDoc.Fields[6].Type = "Certificate" + GCPSEVSNPDoc.Fields[6].Note = "" + GCPSEVSNPDoc.Fields[6].Description = "AMD Signing Key certificate used to verify the SEV-SNP VCEK / VLEK certificate." + GCPSEVSNPDoc.Fields[6].Comments[encoder.LineComment] = "AMD Signing Key certificate used to verify the SEV-SNP VCEK / VLEK certificate." + QEMUVTPMDoc.Type = "QEMUVTPM" QEMUVTPMDoc.Comments[encoder.LineComment] = "QEMUVTPM is the configuration for QEMU vTPM attestation." QEMUVTPMDoc.Description = "QEMUVTPM is the configuration for QEMU vTPM attestation." @@ -580,22 +628,22 @@ func init() { AWSSEVSNPDoc.Fields[0].Description = "Expected TPM measurements." AWSSEVSNPDoc.Fields[0].Comments[encoder.LineComment] = "Expected TPM measurements." AWSSEVSNPDoc.Fields[1].Name = "bootloaderVersion" - AWSSEVSNPDoc.Fields[1].Type = "AttestationVersion" + AWSSEVSNPDoc.Fields[1].Type = "" AWSSEVSNPDoc.Fields[1].Note = "" AWSSEVSNPDoc.Fields[1].Description = "Lowest acceptable bootloader version." AWSSEVSNPDoc.Fields[1].Comments[encoder.LineComment] = "Lowest acceptable bootloader version." AWSSEVSNPDoc.Fields[2].Name = "teeVersion" - AWSSEVSNPDoc.Fields[2].Type = "AttestationVersion" + AWSSEVSNPDoc.Fields[2].Type = "" AWSSEVSNPDoc.Fields[2].Note = "" AWSSEVSNPDoc.Fields[2].Description = "Lowest acceptable TEE version." AWSSEVSNPDoc.Fields[2].Comments[encoder.LineComment] = "Lowest acceptable TEE version." AWSSEVSNPDoc.Fields[3].Name = "snpVersion" - AWSSEVSNPDoc.Fields[3].Type = "AttestationVersion" + AWSSEVSNPDoc.Fields[3].Type = "" AWSSEVSNPDoc.Fields[3].Note = "" AWSSEVSNPDoc.Fields[3].Description = "Lowest acceptable SEV-SNP version." AWSSEVSNPDoc.Fields[3].Comments[encoder.LineComment] = "Lowest acceptable SEV-SNP version." AWSSEVSNPDoc.Fields[4].Name = "microcodeVersion" - AWSSEVSNPDoc.Fields[4].Type = "AttestationVersion" + AWSSEVSNPDoc.Fields[4].Type = "" AWSSEVSNPDoc.Fields[4].Note = "" AWSSEVSNPDoc.Fields[4].Description = "Lowest acceptable microcode version." AWSSEVSNPDoc.Fields[4].Comments[encoder.LineComment] = "Lowest acceptable microcode version." @@ -642,22 +690,22 @@ func init() { AzureSEVSNPDoc.Fields[0].Description = "Expected TPM measurements." AzureSEVSNPDoc.Fields[0].Comments[encoder.LineComment] = "Expected TPM measurements." AzureSEVSNPDoc.Fields[1].Name = "bootloaderVersion" - AzureSEVSNPDoc.Fields[1].Type = "AttestationVersion" + AzureSEVSNPDoc.Fields[1].Type = "" AzureSEVSNPDoc.Fields[1].Note = "" AzureSEVSNPDoc.Fields[1].Description = "Lowest acceptable bootloader version." AzureSEVSNPDoc.Fields[1].Comments[encoder.LineComment] = "Lowest acceptable bootloader version." AzureSEVSNPDoc.Fields[2].Name = "teeVersion" - AzureSEVSNPDoc.Fields[2].Type = "AttestationVersion" + AzureSEVSNPDoc.Fields[2].Type = "" AzureSEVSNPDoc.Fields[2].Note = "" AzureSEVSNPDoc.Fields[2].Description = "Lowest acceptable TEE version." AzureSEVSNPDoc.Fields[2].Comments[encoder.LineComment] = "Lowest acceptable TEE version." AzureSEVSNPDoc.Fields[3].Name = "snpVersion" - AzureSEVSNPDoc.Fields[3].Type = "AttestationVersion" + AzureSEVSNPDoc.Fields[3].Type = "" AzureSEVSNPDoc.Fields[3].Note = "" AzureSEVSNPDoc.Fields[3].Description = "Lowest acceptable SEV-SNP version." AzureSEVSNPDoc.Fields[3].Comments[encoder.LineComment] = "Lowest acceptable SEV-SNP version." AzureSEVSNPDoc.Fields[4].Name = "microcodeVersion" - AzureSEVSNPDoc.Fields[4].Type = "AttestationVersion" + AzureSEVSNPDoc.Fields[4].Type = "" AzureSEVSNPDoc.Fields[4].Note = "" AzureSEVSNPDoc.Fields[4].Description = "Lowest acceptable microcode version." AzureSEVSNPDoc.Fields[4].Comments[encoder.LineComment] = "Lowest acceptable microcode version." @@ -692,6 +740,57 @@ func init() { AzureTrustedLaunchDoc.Fields[0].Note = "" AzureTrustedLaunchDoc.Fields[0].Description = "Expected TPM measurements." AzureTrustedLaunchDoc.Fields[0].Comments[encoder.LineComment] = "Expected TPM measurements." + + AzureTDXDoc.Type = "AzureTDX" + AzureTDXDoc.Comments[encoder.LineComment] = "AzureTDX is the configuration for Azure TDX attestation." + AzureTDXDoc.Description = "AzureTDX is the configuration for Azure TDX attestation." + AzureTDXDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "AttestationConfig", + FieldName: "azureTDX", + }, + } + AzureTDXDoc.Fields = make([]encoder.Doc, 8) + AzureTDXDoc.Fields[0].Name = "measurements" + AzureTDXDoc.Fields[0].Type = "M" + AzureTDXDoc.Fields[0].Note = "" + AzureTDXDoc.Fields[0].Description = "Expected TPM measurements." + AzureTDXDoc.Fields[0].Comments[encoder.LineComment] = "Expected TPM measurements." + AzureTDXDoc.Fields[1].Name = "qeSVN" + AzureTDXDoc.Fields[1].Type = "" + AzureTDXDoc.Fields[1].Note = "" + AzureTDXDoc.Fields[1].Description = "Minimum required QE security version number (SVN)." + AzureTDXDoc.Fields[1].Comments[encoder.LineComment] = "Minimum required QE security version number (SVN)." + AzureTDXDoc.Fields[2].Name = "pceSVN" + AzureTDXDoc.Fields[2].Type = "" + AzureTDXDoc.Fields[2].Note = "" + AzureTDXDoc.Fields[2].Description = "Minimum required PCE security version number (SVN)." + AzureTDXDoc.Fields[2].Comments[encoder.LineComment] = "Minimum required PCE security version number (SVN)." + AzureTDXDoc.Fields[3].Name = "teeTCBSVN" + AzureTDXDoc.Fields[3].Type = "" + AzureTDXDoc.Fields[3].Note = "" + AzureTDXDoc.Fields[3].Description = "Component-wise minimum required 16 byte hex-encoded TEE_TCB security version number (SVN)." + AzureTDXDoc.Fields[3].Comments[encoder.LineComment] = "Component-wise minimum required 16 byte hex-encoded TEE_TCB security version number (SVN)." + AzureTDXDoc.Fields[4].Name = "qeVendorID" + AzureTDXDoc.Fields[4].Type = "" + AzureTDXDoc.Fields[4].Note = "" + AzureTDXDoc.Fields[4].Description = "Expected 16 byte hex-encoded QE_VENDOR_ID field." + AzureTDXDoc.Fields[4].Comments[encoder.LineComment] = "Expected 16 byte hex-encoded QE_VENDOR_ID field." + AzureTDXDoc.Fields[5].Name = "mrSeam" + AzureTDXDoc.Fields[5].Type = "HexBytes" + AzureTDXDoc.Fields[5].Note = "" + AzureTDXDoc.Fields[5].Description = "Expected 48 byte hex-encoded MR_SEAM value." + AzureTDXDoc.Fields[5].Comments[encoder.LineComment] = "Expected 48 byte hex-encoded MR_SEAM value." + AzureTDXDoc.Fields[6].Name = "xfam" + AzureTDXDoc.Fields[6].Type = "" + AzureTDXDoc.Fields[6].Note = "" + AzureTDXDoc.Fields[6].Description = "Expected 8 byte hex-encoded eXtended Features Available Mask (XFAM) field. Defaults to the latest available XFAM on Azure VMs. Unset to disable validation." + AzureTDXDoc.Fields[6].Comments[encoder.LineComment] = "Expected 8 byte hex-encoded eXtended Features Available Mask (XFAM) field. Defaults to the latest available XFAM on Azure VMs. Unset to disable validation." + AzureTDXDoc.Fields[7].Name = "intelRootKey" + AzureTDXDoc.Fields[7].Type = "Certificate" + AzureTDXDoc.Fields[7].Note = "" + AzureTDXDoc.Fields[7].Description = "Intel Root Key certificate used to verify the TDX certificate chain." + AzureTDXDoc.Fields[7].Comments[encoder.LineComment] = "Intel Root Key certificate used to verify the TDX certificate chain." } func (_ Config) Doc() *encoder.Doc { @@ -742,6 +841,10 @@ func (_ GCPSEVES) Doc() *encoder.Doc { return &GCPSEVESDoc } +func (_ GCPSEVSNP) Doc() *encoder.Doc { + return &GCPSEVSNPDoc +} + func (_ QEMUVTPM) Doc() *encoder.Doc { return &QEMUVTPMDoc } @@ -766,6 +869,10 @@ func (_ AzureTrustedLaunch) Doc() *encoder.Doc { return &AzureTrustedLaunchDoc } +func (_ AzureTDX) Doc() *encoder.Doc { + return &AzureTDXDoc +} + // GetConfigurationDoc returns documentation for the file ./config_doc.go. func GetConfigurationDoc() *encoder.FileDoc { return &encoder.FileDoc{ @@ -784,12 +891,14 @@ func GetConfigurationDoc() *encoder.FileDoc { &UnsupportedAppRegistrationErrorDoc, &SNPFirmwareSignerConfigDoc, &GCPSEVESDoc, + &GCPSEVSNPDoc, &QEMUVTPMDoc, &QEMUTDXDoc, &AWSSEVSNPDoc, &AWSNitroTPMDoc, &AzureSEVSNPDoc, &AzureTrustedLaunchDoc, + &AzureTDXDoc, }, } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 21aed1e4c..1c9fbe50f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -34,7 +34,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestDefaultConfig(t *testing.T) { @@ -70,10 +70,10 @@ func TestGetAttestationConfigMarshalsNumericalVersion(t *testing.T) { var mp map[string]interface{} require.NoError(yaml.Unmarshal(bt, &mp)) assert := assert.New(t) - assert.Equal(placeholderVersionValue, mp["microcodeVersion"]) - assert.Equal(placeholderVersionValue, mp["teeVersion"]) - assert.Equal(placeholderVersionValue, mp["snpVersion"]) - assert.Equal(placeholderVersionValue, mp["bootloaderVersion"]) + assert.EqualValues(placeholderVersionValue[uint8](), mp["microcodeVersion"]) + assert.EqualValues(placeholderVersionValue[uint8](), mp["teeVersion"]) + assert.EqualValues(placeholderVersionValue[uint8](), mp["snpVersion"]) + assert.EqualValues(placeholderVersionValue[uint8](), mp["bootloaderVersion"]) } func TestNew(t *testing.T) { @@ -99,19 +99,19 @@ func TestNew(t *testing.T) { wantResult: func() *Config { conf := Default() modifyConfigForAzureToPassValidate(conf) - conf.Attestation.AzureSEVSNP.MicrocodeVersion = AttestationVersion{ + conf.Attestation.AzureSEVSNP.MicrocodeVersion = AttestationVersion[uint8]{ Value: testCfg.Microcode, WantLatest: true, } - conf.Attestation.AzureSEVSNP.TEEVersion = AttestationVersion{ + conf.Attestation.AzureSEVSNP.TEEVersion = AttestationVersion[uint8]{ Value: 2, WantLatest: false, } - conf.Attestation.AzureSEVSNP.BootloaderVersion = AttestationVersion{ + conf.Attestation.AzureSEVSNP.BootloaderVersion = AttestationVersion[uint8]{ Value: 1, WantLatest: false, } - conf.Attestation.AzureSEVSNP.SNPVersion = AttestationVersion{ + conf.Attestation.AzureSEVSNP.SNPVersion = AttestationVersion[uint8]{ Value: testCfg.SNP, WantLatest: true, } @@ -328,12 +328,12 @@ func TestFromFile(t *testing.T) { } func TestValidate(t *testing.T) { - const defaultErrCount = 37 // expect this number of error messages by default because user-specific values are not set and multiple providers are defined by default + const defaultErrCount = 33 // expect this number of error messages by default because user-specific values are not set and multiple providers are defined by default const azErrCount = 7 const awsErrCount = 8 const gcpErrCount = 8 - // TODO(AB#3132,3u13r): refactor config validation tests + // TODO(AB#3132): refactor config validation tests // Note that the `cnf.Image = ""` is a hack to align `bazel test` with `go test` behavior // since first does version stamping. testCases := map[string]struct { @@ -464,9 +464,10 @@ func TestValidate(t *testing.T) { gcp.Project = "test-project" gcp.Zone = "test-zone" gcp.ServiceAccountKeyPath = "test-key-path" + gcp.IAMServiceAccountVM = "example@example.com" cnf.Provider = ProviderConfig{} cnf.Provider.GCP = gcp - cnf.Attestation.GCPSEVES.Measurements = measurements.M{ + cnf.Attestation.GCPSEVSNP.Measurements = measurements.M{ 0: measurements.WithAllBytes(0x00, measurements.Enforce, measurements.PCRMeasurementLength), } cnf.NodeGroups = map[string]NodeGroup{ @@ -624,11 +625,11 @@ func TestConfig_UpdateMeasurements(t *testing.T) { { // GCP conf := Default() conf.RemoveProviderAndAttestationExcept(cloudprovider.GCP) - for k := range conf.Attestation.GCPSEVES.Measurements { - delete(conf.Attestation.GCPSEVES.Measurements, k) + for k := range conf.Attestation.GCPSEVSNP.Measurements { + delete(conf.Attestation.GCPSEVSNP.Measurements, k) } conf.UpdateMeasurements(newMeasurements) - assert.Equal(newMeasurements, conf.Attestation.GCPSEVES.Measurements) + assert.Equal(newMeasurements, conf.Attestation.GCPSEVSNP.Measurements) } { // QEMU conf := Default() @@ -686,140 +687,172 @@ func TestConfig_IsReleaseImage(t *testing.T) { func TestValidInstanceTypeForProvider(t *testing.T) { testCases := map[string]struct { - provider cloudprovider.Provider + variant variant.Variant instanceTypes []string - nonCVMsAllowed bool + providerConfig ProviderConfig expectedResult bool }{ "empty all": { - provider: cloudprovider.Unknown, + variant: variant.Dummy{}, instanceTypes: []string{}, expectedResult: false, + providerConfig: ProviderConfig{}, }, "empty aws": { - provider: cloudprovider.AWS, + variant: variant.AWSSEVSNP{}, instanceTypes: []string{}, expectedResult: false, + providerConfig: ProviderConfig{}, }, "empty azure only CVMs": { - provider: cloudprovider.Azure, + variant: variant.AzureSEVSNP{}, instanceTypes: []string{}, expectedResult: false, + providerConfig: ProviderConfig{}, }, "empty azure with non-CVMs": { - provider: cloudprovider.Azure, + variant: variant.AzureTrustedLaunch{}, instanceTypes: []string{}, - nonCVMsAllowed: true, expectedResult: false, + providerConfig: ProviderConfig{}, }, "empty gcp": { - provider: cloudprovider.GCP, + variant: variant.GCPSEVES{}, instanceTypes: []string{}, expectedResult: false, + providerConfig: ProviderConfig{}, }, - "azure only CVMs": { - provider: cloudprovider.Azure, - instanceTypes: instancetypes.AzureCVMInstanceTypes, + "azure only CVMs (SNP)": { + variant: variant.AzureSEVSNP{}, + instanceTypes: instancetypes.AzureSNPInstanceTypes, expectedResult: true, + providerConfig: ProviderConfig{}, }, - "azure CVMs but CVMs disabled": { - provider: cloudprovider.Azure, - instanceTypes: instancetypes.AzureCVMInstanceTypes, - nonCVMsAllowed: true, - expectedResult: false, - }, - "azure trusted launch VMs with CVMs enabled": { - provider: cloudprovider.Azure, - instanceTypes: instancetypes.AzureTrustedLaunchInstanceTypes, - expectedResult: false, - }, - "azure trusted launch VMs with CVMs disabled": { - provider: cloudprovider.Azure, - instanceTypes: instancetypes.AzureTrustedLaunchInstanceTypes, - nonCVMsAllowed: true, + "azure only CVMs (TDX)": { + variant: variant.AzureTDX{}, + instanceTypes: instancetypes.AzureTDXInstanceTypes, expectedResult: true, + providerConfig: ProviderConfig{}, + }, + "azure trusted launch VMs": { + variant: variant.AzureTrustedLaunch{}, + instanceTypes: instancetypes.AzureTrustedLaunchInstanceTypes, + expectedResult: true, + providerConfig: ProviderConfig{}, }, "gcp": { - provider: cloudprovider.GCP, + variant: variant.GCPSEVES{}, instanceTypes: instancetypes.GCPInstanceTypes, expectedResult: true, + providerConfig: ProviderConfig{}, + }, + "gcp sev-snp": { + variant: variant.GCPSEVSNP{}, + instanceTypes: instancetypes.GCPInstanceTypes, + expectedResult: true, + providerConfig: ProviderConfig{}, }, "put gcp when azure is set": { - provider: cloudprovider.Azure, + variant: variant.AzureSEVSNP{}, instanceTypes: instancetypes.GCPInstanceTypes, expectedResult: false, - }, - "put gcp when azure is set with CVMs disabled": { - provider: cloudprovider.Azure, - instanceTypes: instancetypes.GCPInstanceTypes, - nonCVMsAllowed: true, - expectedResult: false, + providerConfig: ProviderConfig{}, }, "put azure when gcp is set": { - provider: cloudprovider.GCP, - instanceTypes: instancetypes.AzureCVMInstanceTypes, - expectedResult: false, - }, - "put azure when gcp is set with CVMs disabled": { - provider: cloudprovider.GCP, - instanceTypes: instancetypes.AzureTrustedLaunchInstanceTypes, - nonCVMsAllowed: true, + variant: variant.GCPSEVES{}, + instanceTypes: instancetypes.AzureSNPInstanceTypes, expectedResult: false, + providerConfig: ProviderConfig{}, }, // Testing every possible instance type for AWS is not feasible, so we just test a few based on known supported / unsupported families // Also serves as a test for checkIfInstanceInValidAWSFamilys "aws two valid instances": { - provider: cloudprovider.AWS, + variant: variant.AWSSEVSNP{}, instanceTypes: []string{"c5.xlarge", "c5a.2xlarge", "c5a.16xlarge", "u-12tb1.112xlarge"}, - expectedResult: true, - nonCVMsAllowed: true, + expectedResult: false, // False because 2 two of the instances are not valid + providerConfig: ProviderConfig{}, }, "aws one valid instance one with too little vCPUs": { - provider: cloudprovider.AWS, + variant: variant.AWSSEVSNP{}, instanceTypes: []string{"c5.medium"}, expectedResult: false, - nonCVMsAllowed: true, + providerConfig: ProviderConfig{}, }, "aws graviton sub-family unsupported": { - provider: cloudprovider.AWS, + variant: variant.AWSSEVSNP{}, instanceTypes: []string{"m6g.xlarge", "r6g.2xlarge", "x2gd.xlarge", "g5g.8xlarge"}, expectedResult: false, - nonCVMsAllowed: true, + providerConfig: ProviderConfig{}, }, "aws combined two valid instances as one string": { - provider: cloudprovider.AWS, + variant: variant.AWSSEVSNP{}, instanceTypes: []string{"c5.xlarge, c5a.2xlarge"}, expectedResult: false, - nonCVMsAllowed: true, + providerConfig: ProviderConfig{}, }, "aws only CVMs": { - provider: cloudprovider.AWS, + variant: variant.AWSSEVSNP{}, instanceTypes: []string{"c6a.xlarge", "m6a.xlarge", "r6a.xlarge"}, expectedResult: true, + providerConfig: ProviderConfig{}, }, - "aws CVMs but CVMs disabled": { - provider: cloudprovider.AWS, - instanceTypes: []string{"m6a.xlarge", "c6a.xlarge", "r6a.xlarge"}, - nonCVMsAllowed: true, - expectedResult: true, - }, - "aws nitroTPM VMs with CVMs enabled": { - provider: cloudprovider.AWS, + "aws nitroTPM VMs": { + variant: variant.AWSNitroTPM{}, instanceTypes: []string{"c5.xlarge", "c5a.2xlarge", "c5a.16xlarge", "u-12tb1.112xlarge"}, + expectedResult: true, + providerConfig: ProviderConfig{}, + }, + "stackit valid flavors": { + variant: variant.QEMUVTPM{}, + instanceTypes: []string{ + "m1a.2cd", + "m1a.4cd", + "m1a.8cd", + "m1a.16cd", + "m1a.30cd", + }, + expectedResult: true, + providerConfig: ProviderConfig{OpenStack: &OpenStackConfig{Cloud: "stackit"}}, + }, + "stackit not valid flavors": { + variant: variant.QEMUVTPM{}, + instanceTypes: []string{ + // removed the c which indicates a confidential flavor + "m1a.2d", + "m1a.4d", + "m1a.8d", + "m1a.16d", + "m1a.30d", + }, expectedResult: false, + providerConfig: ProviderConfig{OpenStack: &OpenStackConfig{Cloud: "stackit"}}, }, - "aws nitroTPM VMs with CVMs disabled": { - provider: cloudprovider.AWS, - instanceTypes: []string{"c5.xlarge", "c5a.2xlarge", "c5a.16xlarge", "u-12tb1.112xlarge"}, - nonCVMsAllowed: true, + "openstack cloud named test": { + variant: variant.QEMUVTPM{}, + instanceTypes: []string{ + "foo.bar", + "foo.bar1", + }, expectedResult: true, + providerConfig: ProviderConfig{OpenStack: &OpenStackConfig{Cloud: "test"}}, + }, + "Qemutdx valid instance type": { + variant: variant.QEMUTDX{}, + instanceTypes: []string{ + "foo.bar", + }, + expectedResult: true, + providerConfig: ProviderConfig{QEMU: &QEMUConfig{}}, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) for _, instanceType := range tc.instanceTypes { - assert.Equal(tc.expectedResult, validInstanceTypeForProvider(instanceType, tc.nonCVMsAllowed, tc.provider), instanceType) + assert.Equal( + tc.expectedResult, validInstanceTypeForProvider(instanceType, tc.variant, tc.providerConfig), + instanceType, + ) } }) } @@ -1080,18 +1113,8 @@ func getConfigAsMap(conf *Config, t *testing.T) (res configMap) { type stubAttestationFetcher struct{} -func (f stubAttestationFetcher) FetchSEVSNPVersionList(_ context.Context, _ attestationconfigapi.SEVSNPVersionList) (attestationconfigapi.SEVSNPVersionList, error) { - return attestationconfigapi.SEVSNPVersionList{}, nil -} - -func (f stubAttestationFetcher) FetchSEVSNPVersion(_ context.Context, _ attestationconfigapi.SEVSNPVersionAPI) (attestationconfigapi.SEVSNPVersionAPI, error) { - return attestationconfigapi.SEVSNPVersionAPI{ - SEVSNPVersion: testCfg, - }, nil -} - -func (f stubAttestationFetcher) FetchSEVSNPVersionLatest(_ context.Context, _ variant.Variant) (attestationconfigapi.SEVSNPVersionAPI, error) { - return attestationconfigapi.SEVSNPVersionAPI{ +func (f stubAttestationFetcher) FetchLatestVersion(_ context.Context, _ variant.Variant) (attestationconfigapi.Entry, error) { + return attestationconfigapi.Entry{ SEVSNPVersion: testCfg, }, nil } diff --git a/internal/config/disktypes/aws.go b/internal/config/disktypes/aws.go index 8fcdc354a..bb0e0586b 100644 --- a/internal/config/disktypes/aws.go +++ b/internal/config/disktypes/aws.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package disktypes diff --git a/internal/config/disktypes/azure.go b/internal/config/disktypes/azure.go index 8903a45d2..94078b07f 100644 --- a/internal/config/disktypes/azure.go +++ b/internal/config/disktypes/azure.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package disktypes diff --git a/internal/config/disktypes/gcp.go b/internal/config/disktypes/gcp.go index 3880b9a2f..cfb9315c2 100644 --- a/internal/config/disktypes/gcp.go +++ b/internal/config/disktypes/gcp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package disktypes diff --git a/internal/config/gcp.go b/internal/config/gcp.go new file mode 100644 index 000000000..79fe43399 --- /dev/null +++ b/internal/config/gcp.go @@ -0,0 +1,128 @@ +/* +Copyright (c) Edgeless Systems GmbH +SPDX-License-Identifier: BUSL-1.1 +*/ + +package config + +import ( + "bytes" + "context" + "fmt" + + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/edgelesssys/constellation/v2/internal/attestation/measurements" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" +) + +var _ svnResolveMarshaller = &GCPSEVSNP{} + +// DefaultForGCPSEVSNP provides a valid default configuration for GCP SEV-SNP attestation. +func DefaultForGCPSEVSNP() *GCPSEVSNP { + return &GCPSEVSNP{ + Measurements: measurements.DefaultsFor(cloudprovider.GCP, variant.GCPSEVSNP{}), + BootloaderVersion: NewLatestPlaceholderVersion[uint8](), + TEEVersion: NewLatestPlaceholderVersion[uint8](), + SNPVersion: NewLatestPlaceholderVersion[uint8](), + MicrocodeVersion: NewLatestPlaceholderVersion[uint8](), + AMDRootKey: mustParsePEM(arkPEM), + } +} + +// GetVariant returns gcp-sev-snp as the variant. +func (GCPSEVSNP) GetVariant() variant.Variant { + return variant.GCPSEVSNP{} +} + +// GetMeasurements returns the measurements used for attestation. +func (c GCPSEVSNP) GetMeasurements() measurements.M { + return c.Measurements +} + +// SetMeasurements updates a config's measurements using the given measurements. +func (c *GCPSEVSNP) SetMeasurements(m measurements.M) { + c.Measurements = m +} + +// EqualTo returns true if the config is equal to the given config. +func (c GCPSEVSNP) EqualTo(other AttestationCfg) (bool, error) { + otherCfg, ok := other.(*GCPSEVSNP) + if !ok { + return false, fmt.Errorf("cannot compare %T with %T", c, other) + } + + measurementsEqual := c.Measurements.EqualTo(otherCfg.Measurements) + bootloaderEqual := c.BootloaderVersion == otherCfg.BootloaderVersion + teeEqual := c.TEEVersion == otherCfg.TEEVersion + snpEqual := c.SNPVersion == otherCfg.SNPVersion + microcodeEqual := c.MicrocodeVersion == otherCfg.MicrocodeVersion + rootKeyEqual := bytes.Equal(c.AMDRootKey.Raw, otherCfg.AMDRootKey.Raw) + signingKeyEqual := bytes.Equal(c.AMDSigningKey.Raw, otherCfg.AMDSigningKey.Raw) + + return measurementsEqual && bootloaderEqual && teeEqual && snpEqual && microcodeEqual && rootKeyEqual && signingKeyEqual, nil +} + +func (c *GCPSEVSNP) getToMarshallLatestWithResolvedVersions() AttestationCfg { + cp := *c + cp.BootloaderVersion.WantLatest = false + cp.TEEVersion.WantLatest = false + cp.SNPVersion.WantLatest = false + cp.MicrocodeVersion.WantLatest = false + return &cp +} + +// FetchAndSetLatestVersionNumbers fetches the latest version numbers from the configapi and sets them. +func (c *GCPSEVSNP) FetchAndSetLatestVersionNumbers(ctx context.Context, fetcher attestationconfigapi.Fetcher) error { + // Only talk to the API if at least one version number is set to latest. + if !(c.BootloaderVersion.WantLatest || c.TEEVersion.WantLatest || c.SNPVersion.WantLatest || c.MicrocodeVersion.WantLatest) { + return nil + } + + versions, err := fetcher.FetchLatestVersion(ctx, variant.GCPSEVSNP{}) + if err != nil { + return fmt.Errorf("fetching latest TCB versions from configapi: %w", err) + } + // set number and keep isLatest flag + c.mergeWithLatestVersion(versions.SEVSNPVersion) + return nil +} + +func (c *GCPSEVSNP) mergeWithLatestVersion(latest attestationconfigapi.SEVSNPVersion) { + if c.BootloaderVersion.WantLatest { + c.BootloaderVersion.Value = latest.Bootloader + } + if c.TEEVersion.WantLatest { + c.TEEVersion.Value = latest.TEE + } + if c.SNPVersion.WantLatest { + c.SNPVersion.Value = latest.SNP + } + if c.MicrocodeVersion.WantLatest { + c.MicrocodeVersion.Value = latest.Microcode + } +} + +// GetVariant returns gcp-sev-es as the variant. +func (GCPSEVES) GetVariant() variant.Variant { + return variant.GCPSEVES{} +} + +// GetMeasurements returns the measurements used for attestation. +func (c GCPSEVES) GetMeasurements() measurements.M { + return c.Measurements +} + +// SetMeasurements updates a config's measurements using the given measurements. +func (c *GCPSEVES) SetMeasurements(m measurements.M) { + c.Measurements = m +} + +// EqualTo returns true if the config is equal to the given config. +func (c GCPSEVES) EqualTo(other AttestationCfg) (bool, error) { + otherCfg, ok := other.(*GCPSEVES) + if !ok { + return false, fmt.Errorf("cannot compare %T with %T", c, other) + } + return c.Measurements.EqualTo(otherCfg.Measurements), nil +} diff --git a/internal/config/image_enterprise.go b/internal/config/image_enterprise.go index e92936da6..85fef6c52 100644 --- a/internal/config/image_enterprise.go +++ b/internal/config/image_enterprise.go @@ -3,12 +3,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config const ( // defaultImage is the default image to use. - defaultImage = "ref/main/stream/nightly/v2.14.0-pre.0.20231214193540-2c50abcc919b" + defaultImage = "ref/main/stream/nightly/v2.24.0-pre.0.20250731155507-57874454f72c" ) diff --git a/internal/config/image_oss.go b/internal/config/image_oss.go index 939c16995..dc9141a8e 100644 --- a/internal/config/image_oss.go +++ b/internal/config/image_oss.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config diff --git a/internal/config/imageversion/imageversion.go b/internal/config/imageversion/imageversion.go index c295c40f4..9e968715a 100644 --- a/internal/config/imageversion/imageversion.go +++ b/internal/config/imageversion/imageversion.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package imageversion contains the pinned container images for the config. diff --git a/internal/config/imageversion/placeholder.go b/internal/config/imageversion/placeholder.go index f5ba25ea4..a0273b96f 100644 --- a/internal/config/imageversion/placeholder.go +++ b/internal/config/imageversion/placeholder.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package imageversion diff --git a/internal/config/instancetypes/BUILD.bazel b/internal/config/instancetypes/BUILD.bazel index 7080f5bb8..609892693 100644 --- a/internal/config/instancetypes/BUILD.bazel +++ b/internal/config/instancetypes/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "aws.go", "azure.go", "gcp.go", + "stackit.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/config/instancetypes", visibility = ["//:__subpackages__"], diff --git a/internal/config/instancetypes/aws.go b/internal/config/instancetypes/aws.go index 712cc4f86..edafba99d 100644 --- a/internal/config/instancetypes/aws.go +++ b/internal/config/instancetypes/aws.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package instancetypes diff --git a/internal/config/instancetypes/azure.go b/internal/config/instancetypes/azure.go index 29525351d..483e950bd 100644 --- a/internal/config/instancetypes/azure.go +++ b/internal/config/instancetypes/azure.go @@ -1,14 +1,14 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package instancetypes -// AzureCVMInstanceTypes are valid Azure CVM instance types. -var AzureCVMInstanceTypes = []string{ - // CVMs (3rd Generation EPYC 7763v processors) +// AzureSNPInstanceTypes are valid Azure SEV-SNP instance types. +var AzureSNPInstanceTypes = []string{ + // SEV-SNP CVMs (3rd Generation EPYC 7763v processors) // DCasv5-series "Standard_DC4as_v5", "Standard_DC8as_v5", @@ -45,6 +45,43 @@ var AzureCVMInstanceTypes = []string{ "Standard_EC96ads_v5", } +// AzureTDXInstanceTypes are valid Azure TDX instance types. +var AzureTDXInstanceTypes = []string{ + // TDX CVMs + // DCesv5-series + "Standard_DC4es_v5", + "Standard_DC8es_v5", + "Standard_DC16es_v5", + "Standard_DC32es_v5", + "Standard_DC48es_v5", + "Standard_DC64es_v5", + "Standard_DC96es_v5", + // DCedsv5-series + "Standard_DC4eds_v5", + "Standard_DC8eds_v5", + "Standard_DC16eds_v5", + "Standard_DC32eds_v5", + "Standard_DC48eds_v5", + "Standard_DC64eds_v5", + "Standard_DC96eds_v5", + // ECesv5-series + "Standard_EC4es_v5", + "Standard_EC8es_v5", + "Standard_EC16es_v5", + "Standard_EC32es_v5", + "Standard_EC48es_v5", + "Standard_EC64es_v5", + "Standard_EC128es_v5", + // ECedsv5-series + "Standard_EC4eds_v5", + "Standard_EC8eds_v5", + "Standard_EC16eds_v5", + "Standard_EC32eds_v5", + "Standard_EC48eds_v5", + "Standard_EC64eds_v5", + "Standard_EC128eds_v5", +} + // AzureTrustedLaunchInstanceTypes are valid Azure Trusted Launch instance types. var AzureTrustedLaunchInstanceTypes = []string{ // Trusted Launch (2nd Generation AMD EPYC 7452 or 3rd Generation EPYC 7763v processors) diff --git a/internal/config/instancetypes/gcp.go b/internal/config/instancetypes/gcp.go index c9d02a345..5d85a9a60 100644 --- a/internal/config/instancetypes/gcp.go +++ b/internal/config/instancetypes/gcp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package instancetypes diff --git a/internal/config/instancetypes/stackit.go b/internal/config/instancetypes/stackit.go new file mode 100644 index 000000000..83ab851c1 --- /dev/null +++ b/internal/config/instancetypes/stackit.go @@ -0,0 +1,16 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package instancetypes + +// STACKITInstanceTypes are valid STACKIT instance types. +var STACKITInstanceTypes = []string{ + "m1a.2cd", + "m1a.4cd", + "m1a.8cd", + "m1a.16cd", + "m1a.30cd", +} diff --git a/internal/config/migration/migration.go b/internal/config/migration/migration.go index 8b8661033..4799f162f 100644 --- a/internal/config/migration/migration.go +++ b/internal/config/migration/migration.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package migration contains outdated configuration formats and their migration functions. @@ -140,10 +140,6 @@ type AWSSEVSNP struct { // description: | // Expected TPM measurements. Measurements measurements.M `json:"measurements" yaml:"measurements" validate:"required,no_placeholders"` - // TODO(derpsteb): reenable launchMeasurement once SNP is fixed on AWS. - // description: | - // Expected launch measurement in SNP report. - // LaunchMeasurement measurements.Measurement `json:"launchMeasurement" yaml:"launchMeasurement" validate:"required"` } // AWSNitroTPM is the configuration for AWS Nitro TPM attestation. @@ -381,15 +377,7 @@ func V3ToV4(path string, fileHandler file.Handler) error { Cloud: cfgV3.Provider.OpenStack.Cloud, AvailabilityZone: cfgV3.Provider.OpenStack.AvailabilityZone, FloatingIPPoolID: cfgV3.Provider.OpenStack.FloatingIPPoolID, - AuthURL: cfgV3.Provider.OpenStack.AuthURL, - ProjectID: cfgV3.Provider.OpenStack.ProjectID, - ProjectName: cfgV3.Provider.OpenStack.ProjectName, - UserDomainName: cfgV3.Provider.OpenStack.UserDomainName, - ProjectDomainName: cfgV3.Provider.OpenStack.ProjectDomainName, RegionName: cfgV3.Provider.OpenStack.RegionName, - Username: cfgV3.Provider.OpenStack.Username, - Password: cfgV3.Provider.OpenStack.Password, - DirectDownload: cfgV3.Provider.OpenStack.DirectDownload, DeployYawolLoadBalancer: cfgV3.Provider.OpenStack.DeployYawolLoadBalancer, YawolImageID: cfgV3.Provider.OpenStack.YawolImageID, YawolFlavorID: cfgV3.Provider.OpenStack.YawolFlavorID, @@ -423,19 +411,19 @@ func V3ToV4(path string, fileHandler file.Handler) error { case cfgV3.Attestation.AzureSEVSNP != nil: cfgV4.Attestation.AzureSEVSNP = &config.AzureSEVSNP{ Measurements: cfgV3.Attestation.AzureSEVSNP.Measurements, - BootloaderVersion: config.AttestationVersion{ + BootloaderVersion: config.AttestationVersion[uint8]{ Value: cfgV3.Attestation.AzureSEVSNP.BootloaderVersion.Value, WantLatest: cfgV3.Attestation.AzureSEVSNP.BootloaderVersion.WantLatest, }, - TEEVersion: config.AttestationVersion{ + TEEVersion: config.AttestationVersion[uint8]{ Value: cfgV3.Attestation.AzureSEVSNP.TEEVersion.Value, WantLatest: cfgV3.Attestation.AzureSEVSNP.TEEVersion.WantLatest, }, - SNPVersion: config.AttestationVersion{ + SNPVersion: config.AttestationVersion[uint8]{ Value: cfgV3.Attestation.AzureSEVSNP.SNPVersion.Value, WantLatest: cfgV3.Attestation.AzureSEVSNP.SNPVersion.WantLatest, }, - MicrocodeVersion: config.AttestationVersion{ + MicrocodeVersion: config.AttestationVersion[uint8]{ Value: cfgV3.Attestation.AzureSEVSNP.MicrocodeVersion.Value, WantLatest: cfgV3.Attestation.AzureSEVSNP.MicrocodeVersion.WantLatest, }, diff --git a/internal/config/validation.go b/internal/config/validation.go index 35a55727f..68a7bf821 100644 --- a/internal/config/validation.go +++ b/internal/config/validation.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -193,12 +193,18 @@ func validateAttestation(sl validator.StructLevel) { if attestation.AzureSEVSNP != nil { attestationCount++ } + if attestation.AzureTDX != nil { + attestationCount++ + } if attestation.AzureTrustedLaunch != nil { attestationCount++ } if attestation.GCPSEVES != nil { attestationCount++ } + if attestation.GCPSEVSNP != nil { + attestationCount++ + } if attestation.QEMUVTPM != nil { attestationCount++ } @@ -244,7 +250,7 @@ func translateNoAttestationError(ut ut.Translator, fe validator.FieldError) stri } func registerNoAttestationError(ut ut.Translator) error { - return ut.Add("no_attestation", "{0}: No attestation has been defined (requires either awsSEVSNP, awsNitroTPM, azureSEVSNP, azureTrustedLaunch, gcpSEVES, or qemuVTPM)", true) + return ut.Add("no_attestation", "{0}: No attestation has been defined (requires either awsSEVSNP, awsNitroTPM, azureSEVSNP, azureTDX, azureTrustedLaunch, gcpSEVES, gcpSEVSNP, or qemuVTPM)", true) } func translateNoDefaultControlPlaneGroupError(ut ut.Translator, fe validator.FieldError) string { @@ -352,12 +358,18 @@ func (c *Config) translateMoreThanOneAttestationError(ut ut.Translator, fe valid if c.Attestation.AzureSEVSNP != nil { definedAttestations = append(definedAttestations, "AzureSEVSNP") } + if c.Attestation.AzureTDX != nil { + definedAttestations = append(definedAttestations, "AzureTDX") + } if c.Attestation.AzureTrustedLaunch != nil { definedAttestations = append(definedAttestations, "AzureTrustedLaunch") } if c.Attestation.GCPSEVES != nil { definedAttestations = append(definedAttestations, "GCPSEVES") } + if c.Attestation.GCPSEVSNP != nil { + definedAttestations = append(definedAttestations, "GCPSEVSNP") + } if c.Attestation.QEMUVTPM != nil { definedAttestations = append(definedAttestations, "QEMUVTPM") } @@ -452,9 +464,14 @@ func (c *Config) translateAzureInstanceTypeError(ut ut.Translator, fe validator. attestVariant := c.GetAttestationConfig().GetVariant() - instances := instancetypes.AzureCVMInstanceTypes - if attestVariant.Equal(variant.AzureTrustedLaunch{}) { + var instances []string + switch attestVariant.String() { + case variant.AzureTrustedLaunch{}.String(): instances = instancetypes.AzureTrustedLaunchInstanceTypes + case variant.AzureSEVSNP{}.String(): + instances = instancetypes.AzureSNPInstanceTypes + case variant.AzureTDX{}.String(): + instances = instancetypes.AzureTDXInstanceTypes } t, _ = ut.T("instance_type", fe.Field(), fmt.Sprintf("%v", instances)) @@ -503,37 +520,49 @@ func (c *Config) translateMoreThanOneProviderError(ut ut.Translator, fe validato return t } -func validInstanceTypeForProvider(insType string, acceptNonCVM bool, provider cloudprovider.Provider) bool { - switch provider { - case cloudprovider.AWS: - return isSupportedAWSInstanceType(insType, acceptNonCVM) - case cloudprovider.Azure: - if acceptNonCVM { - for _, instanceType := range instancetypes.AzureTrustedLaunchInstanceTypes { - if insType == instanceType { - return true - } - } - } else { - for _, instanceType := range instancetypes.AzureCVMInstanceTypes { - if insType == instanceType { - return true - } +func validInstanceTypeForProvider(insType string, attestation variant.Variant, provider ProviderConfig) bool { + switch attestation { + case variant.AWSSEVSNP{}, variant.AWSNitroTPM{}: + return isSupportedAWSInstanceType(insType, attestation.Equal(variant.AWSNitroTPM{})) + case variant.AzureSEVSNP{}: + for _, instanceType := range instancetypes.AzureSNPInstanceTypes { + if insType == instanceType { + return true } } - return false - case cloudprovider.GCP: + case variant.AzureTDX{}: + for _, instanceType := range instancetypes.AzureTDXInstanceTypes { + if insType == instanceType { + return true + } + } + case variant.AzureTrustedLaunch{}: + for _, instanceType := range instancetypes.AzureTrustedLaunchInstanceTypes { + if insType == instanceType { + return true + } + } + case variant.GCPSEVES{}, variant.GCPSEVSNP{}: for _, instanceType := range instancetypes.GCPInstanceTypes { if insType == instanceType { return true } } - return false - case cloudprovider.OpenStack, cloudprovider.QEMU: + case variant.QEMUVTPM{}, variant.QEMUTDX{}: + // only allow confidential instances on stackit cloud using QEMU vTPM + if provider.OpenStack != nil { + if cloud := provider.OpenStack.Cloud; strings.ToLower(cloud) == "stackit" { + for _, instanceType := range instancetypes.STACKITInstanceTypes { + if insType == instanceType { + return true + } + } + return false + } + } return true - default: - return false } + return false } // isSupportedAWSInstanceType checks if an AWS instance type passed as user input is in one of the supported instance types. @@ -690,7 +719,8 @@ func msgFromCompatibilityError(err error, binaryVersion, fieldValue string) stri } } -func validateMicroserviceVersion(binaryVersion, version consemver.Semver) error { +// ValidateMicroserviceVersion checks that the version of the microservice is compatible with the binary version. +func ValidateMicroserviceVersion(binaryVersion, version consemver.Semver) error { // Major versions always have to match. if binaryVersion.Major() != version.Major() { return compatibility.ErrMajorMismatch @@ -770,16 +800,7 @@ func (c *Config) validateNodeGroupZoneField(fl validator.FieldLevel) bool { } func (c *Config) validateInstanceType(fl validator.FieldLevel) bool { - acceptNonCVM := false - - if c.GetAttestationConfig().GetVariant().Equal(variant.AzureTrustedLaunch{}) { - acceptNonCVM = true - } - if c.GetAttestationConfig().GetVariant().Equal(variant.AWSNitroTPM{}) { - acceptNonCVM = true - } - - return validInstanceTypeForProvider(fl.Field().String(), acceptNonCVM, c.GetProvider()) + return validInstanceTypeForProvider(fl.Field().String(), c.GetAttestationConfig().GetVariant(), c.Provider) } func (c *Config) validateStateDiskTypeField(fl validator.FieldLevel) bool { diff --git a/internal/config/validation_test.go b/internal/config/validation_test.go index f6a5bde95..2cf9dbc8a 100644 --- a/internal/config/validation_test.go +++ b/internal/config/validation_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package config @@ -75,7 +75,7 @@ func TestValidateMicroserviceVersion(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - err := validateMicroserviceVersion(tc.cli, tc.services) + err := ValidateMicroserviceVersion(tc.cli, tc.services) if tc.wantError { assert.Error(err) return diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 68d0fd6e2..56fca9ef7 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -40,6 +40,16 @@ const ( DefaultControlPlaneGroupName = "control_plane_default" // DefaultWorkerGroupName is the name of the default worker node group. DefaultWorkerGroupName = "worker_default" + // CLIDebugLogFile is the name of the debug log file for constellation init/constellation apply. + CLIDebugLogFile = "constellation-debug.log" + // SSHCAKeySuffix is the suffix used together with the DEKPrefix to derive an SSH CA key for emergency ssh access. + SSHCAKeySuffix = "ca_emergency_ssh" + // SSHCAKeyPath is the path to the emergency SSH CA key on the node. + SSHCAKeyPath = "/var/run/state/ssh/ssh_ca.pub" + // SSHHostKeyPath is the path to the SSH host key of the node. + SSHHostKeyPath = "/var/run/state/ssh/ssh_host_ed25519_key" + // SSHHostCertificatePath is the path to the SSH host certificate. + SSHHostCertificatePath = "/var/run/state/ssh/ssh_host_cert.pub" // // Ports. diff --git a/internal/constants/enterprise.go b/internal/constants/enterprise.go index 7e219eb95..9c4d7421c 100644 --- a/internal/constants/enterprise.go +++ b/internal/constants/enterprise.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constants diff --git a/internal/constants/oss.go b/internal/constants/oss.go index 505084c59..122d3e8f8 100644 --- a/internal/constants/oss.go +++ b/internal/constants/oss.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constants diff --git a/internal/constellation/BUILD.bazel b/internal/constellation/BUILD.bazel index 58bd5d235..02afff0e9 100644 --- a/internal/constellation/BUILD.bazel +++ b/internal/constellation/BUILD.bazel @@ -37,8 +37,13 @@ go_library( "//internal/semver", "//internal/versions", "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions", + "@io_k8s_apimachinery//pkg/api/errors", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/types", + "@io_k8s_client_go//dynamic", "@io_k8s_client_go//tools/clientcmd", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", ], ) @@ -69,6 +74,6 @@ go_test( "@com_github_stretchr_testify//require", "@io_k8s_client_go//tools/clientcmd", "@io_k8s_client_go//tools/clientcmd/api", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", ], ) diff --git a/internal/constellation/apply.go b/internal/constellation/apply.go index 63751379e..6c541cb3b 100644 --- a/internal/constellation/apply.go +++ b/internal/constellation/apply.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation @@ -18,6 +18,18 @@ import ( "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/license" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/clientcmd" +) + +// ApplyContext denotes the context in which the apply command is run. +type ApplyContext string + +const ( + // ApplyContextCLI is used when the Applier is used by the CLI. + ApplyContextCLI ApplyContext = "cli" + // ApplyContextTerraform is used when the Applier is used by Terraform. + ApplyContextTerraform ApplyContext = "terraform" ) // An Applier handles applying a specific configuration to a Constellation cluster @@ -28,30 +40,34 @@ type Applier struct { licenseChecker licenseChecker spinner spinnerInterf + applyContext ApplyContext + // newDialer creates a new aTLS gRPC dialer. newDialer func(validator atls.Validator) *dialer.Dialer kubecmdClient kubecmdClient helmClient helmApplier + dynamicClient dynamic.Interface } type licenseChecker interface { - CheckLicense(context.Context, cloudprovider.Provider, string) (license.QuotaCheckResponse, error) + CheckLicense(ctx context.Context, csp cloudprovider.Provider, action license.Action, licenseID string) (int, error) } type debugLog interface { - Debugf(format string, args ...any) + Debug(msg string, args ...any) } // NewApplier creates a new Applier. func NewApplier( - log debugLog, - spinner spinnerInterf, + log debugLog, spinner spinnerInterf, + applyContext ApplyContext, newDialer func(validator atls.Validator) *dialer.Dialer, ) *Applier { return &Applier{ log: log, spinner: spinner, - licenseChecker: license.NewChecker(license.NewClient()), + licenseChecker: license.NewChecker(), + applyContext: applyContext, newDialer: newDialer, } } @@ -66,27 +82,47 @@ func (a *Applier) SetKubeConfig(kubeConfig []byte) error { if err != nil { return err } + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig) + if err != nil { + return err + } + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return err + } a.kubecmdClient = kubecmdClient a.helmClient = helmClient + a.dynamicClient = dynamicClient return nil } // CheckLicense checks the given Constellation license with the license server // and returns the allowed quota for the license. -func (a *Applier) CheckLicense(ctx context.Context, csp cloudprovider.Provider, licenseID string) (int, error) { - a.log.Debugf("Contacting license server for license '%s'", licenseID) - quotaResp, err := a.licenseChecker.CheckLicense(ctx, csp, licenseID) +func (a *Applier) CheckLicense(ctx context.Context, csp cloudprovider.Provider, initRequest bool, licenseID string) (int, error) { + a.log.Debug(fmt.Sprintf("Contacting license server for license %q", licenseID)) + + var action license.Action + if initRequest { + action = license.Init + } else { + action = license.Apply + } + if a.applyContext == ApplyContextTerraform { + action += "-terraform" + } + + quota, err := a.licenseChecker.CheckLicense(ctx, csp, action, licenseID) if err != nil { return 0, fmt.Errorf("checking license: %w", err) } - a.log.Debugf("Got response from license server for license '%s'", licenseID) + a.log.Debug(fmt.Sprintf("Got response from license server for license %q", licenseID)) - return quotaResp.Quota, nil + return quota, nil } // GenerateMasterSecret generates a new master secret. func (a *Applier) GenerateMasterSecret() (uri.MasterSecret, error) { - a.log.Debugf("Generating master secret") + a.log.Debug("Generating master secret") key, err := crypto.GenerateRandomBytes(crypto.MasterSecretLengthDefault) if err != nil { return uri.MasterSecret{}, err @@ -99,17 +135,17 @@ func (a *Applier) GenerateMasterSecret() (uri.MasterSecret, error) { Key: key, Salt: salt, } - a.log.Debugf("Generated master secret key and salt values") + a.log.Debug("Generated master secret key and salt values") return secret, nil } // GenerateMeasurementSalt generates a new measurement salt. func (a *Applier) GenerateMeasurementSalt() ([]byte, error) { - a.log.Debugf("Generating measurement salt") + a.log.Debug("Generating measurement salt") measurementSalt, err := crypto.GenerateRandomBytes(crypto.RNGLengthDefault) if err != nil { return nil, fmt.Errorf("generating measurement salt: %w", err) } - a.log.Debugf("Generated measurement salt") + a.log.Debug("Generated measurement salt") return measurementSalt, nil } diff --git a/internal/constellation/apply_test.go b/internal/constellation/apply_test.go index ce2466fdf..c7864a7b2 100644 --- a/internal/constellation/apply_test.go +++ b/internal/constellation/apply_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation @@ -38,7 +38,7 @@ func TestCheckLicense(t *testing.T) { require := require.New(t) a := &Applier{licenseChecker: tc.licenseChecker, log: logger.NewTest(t)} - _, err := a.CheckLicense(context.Background(), cloudprovider.Unknown, license.CommunityLicense) + _, err := a.CheckLicense(t.Context(), cloudprovider.Unknown, true, license.CommunityLicense) if tc.wantErr { require.Error(err) } else { @@ -52,8 +52,8 @@ type stubLicenseChecker struct { checkLicenseErr error } -func (c *stubLicenseChecker) CheckLicense(context.Context, cloudprovider.Provider, string) (license.QuotaCheckResponse, error) { - return license.QuotaCheckResponse{}, c.checkLicenseErr +func (c *stubLicenseChecker) CheckLicense(context.Context, cloudprovider.Provider, license.Action, string) (int, error) { + return 0, c.checkLicenseErr } func TestGenerateMasterSecret(t *testing.T) { diff --git a/internal/constellation/applyinit.go b/internal/constellation/applyinit.go index cbec0cc2c..05a9b1e39 100644 --- a/internal/constellation/applyinit.go +++ b/internal/constellation/applyinit.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation @@ -42,7 +42,7 @@ type InitPayload struct { // GrpcDialer dials a gRPC server. type GrpcDialer interface { - Dial(ctx context.Context, target string) (*grpc.ClientConn, error) + Dial(target string) (*grpc.ClientConn, error) } // Init performs the init RPC. @@ -85,21 +85,21 @@ func (a *Applier) Init( // Create a wrapper function that allows logging any returned error from the retrier before checking if it's the expected retriable one. serviceIsUnavailable := func(err error) bool { isServiceUnavailable := grpcRetry.ServiceIsUnavailable(err) - a.log.Debugf("Encountered error (retriable: %t): %s", isServiceUnavailable, err) + a.log.Debug(fmt.Sprintf("Encountered error (retriable: %t): %q", isServiceUnavailable, err)) return isServiceUnavailable } // Perform the RPC - a.log.Debugf("Making initialization call, doer is %+v", doer) + a.log.Debug("Initialization call", "endpoint", doer.endpoint) a.spinner.Start("Connecting ", false) retrier := retry.NewIntervalRetrier(doer, 30*time.Second, serviceIsUnavailable) if err := retrier.Do(ctx); err != nil { return InitOutput{}, fmt.Errorf("doing init call: %w", err) } a.spinner.Stop() - a.log.Debugf("Initialization request finished") + a.log.Debug("Initialization request finished") - a.log.Debugf("Rewriting cluster server address in kubeconfig to %s", state.Infrastructure.ClusterEndpoint) + a.log.Debug(fmt.Sprintf("Rewriting cluster server address in kubeconfig to %q", state.Infrastructure.ClusterEndpoint)) kubeconfig, err := clientcmd.Load(doer.resp.Kubeconfig) if err != nil { return InitOutput{}, fmt.Errorf("loading kubeconfig: %w", err) @@ -173,9 +173,9 @@ func (d *initDoer) Do(ctx context.Context) error { } } - conn, err := d.dialer.Dial(ctx, d.endpoint) + conn, err := d.dialer.Dial(d.endpoint) if err != nil { - d.log.Debugf("Dialing init server failed: %s. Retrying...", err) + d.log.Debug(fmt.Sprintf("Dialing init server failed: %q. Retrying...", err)) return fmt.Errorf("dialing init server: %w", err) } defer conn.Close() @@ -188,7 +188,7 @@ func (d *initDoer) Do(ctx context.Context) error { d.handleGRPCStateChanges(grpcStateLogCtx, &wg, conn) protoClient := initproto.NewAPIClient(conn) - d.log.Debugf("Created protoClient") + d.log.Debug("Created protoClient") resp, err := protoClient.Init(ctx, d.req) if err != nil { return &NonRetriableInitError{ @@ -200,7 +200,7 @@ func (d *initDoer) Do(ctx context.Context) error { res, err := resp.Recv() // get first response, either success or failure if err != nil { if e := d.getLogs(resp); e != nil { - d.log.Debugf("Failed to collect logs: %s", e) + d.log.Debug(fmt.Sprintf("Failed to collect logs: %q", e)) return &NonRetriableInitError{ LogCollectionErr: e, Err: err, @@ -214,7 +214,7 @@ func (d *initDoer) Do(ctx context.Context) error { d.resp = res.GetInitSuccess() case *initproto.InitResponse_InitFailure: if e := d.getLogs(resp); e != nil { - d.log.Debugf("Failed to get logs from cluster: %s", e) + d.log.Debug(fmt.Sprintf("Failed to get logs from cluster: %q", e)) return &NonRetriableInitError{ LogCollectionErr: e, Err: errors.New(res.GetInitFailure().GetError()), @@ -222,10 +222,10 @@ func (d *initDoer) Do(ctx context.Context) error { } return &NonRetriableInitError{Err: errors.New(res.GetInitFailure().GetError())} case nil: - d.log.Debugf("Cluster returned nil response type") + d.log.Debug("Cluster returned nil response type") err = errors.New("empty response from cluster") if e := d.getLogs(resp); e != nil { - d.log.Debugf("Failed to collect logs: %s", e) + d.log.Debug(fmt.Sprintf("Failed to collect logs: %q", e)) return &NonRetriableInitError{ LogCollectionErr: e, Err: err, @@ -233,10 +233,10 @@ func (d *initDoer) Do(ctx context.Context) error { } return &NonRetriableInitError{Err: err} default: - d.log.Debugf("Cluster returned unknown response type") + d.log.Debug("Cluster returned unknown response type") err = errors.New("unknown response from cluster") if e := d.getLogs(resp); e != nil { - d.log.Debugf("Failed to collect logs: %s", e) + d.log.Debug(fmt.Sprintf("Failed to collect logs: %q", e)) return &NonRetriableInitError{ LogCollectionErr: e, Err: err, @@ -249,7 +249,7 @@ func (d *initDoer) Do(ctx context.Context) error { // getLogs retrieves the cluster logs from the bootstrapper and saves them in the initDoer. func (d *initDoer) getLogs(resp initproto.API_InitClient) error { - d.log.Debugf("Attempting to collect cluster logs") + d.log.Debug("Attempting to collect cluster logs") for { res, err := resp.Recv() if err == io.EOF { @@ -277,7 +277,7 @@ func (d *initDoer) getLogs(resp initproto.API_InitClient) error { } } - d.log.Debugf("Received cluster logs") + d.log.Debug("Received cluster logs") return nil } diff --git a/internal/constellation/applyinit_test.go b/internal/constellation/applyinit_test.go index 7d16d5fe7..59025bad2 100644 --- a/internal/constellation/applyinit_test.go +++ b/internal/constellation/applyinit_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation @@ -214,7 +214,7 @@ func TestInit(t *testing.T) { } clusterLogs := &bytes.Buffer{} - ctx, cancel := context.WithTimeout(context.Background(), time.Second*4) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*4) defer cancel() _, err := a.Init(ctx, nil, tc.state, clusterLogs, InitPayload{ MasterSecret: uri.MasterSecret{}, @@ -280,7 +280,7 @@ func TestAttestation(t *testing.T) { } state := &state.State{Version: state.Version1, Infrastructure: state.Infrastructure{ClusterEndpoint: "192.0.2.4"}} - ctx := context.Background() + ctx := t.Context() ctx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() diff --git a/internal/constellation/constellation.go b/internal/constellation/constellation.go index af0fb6b4c..27202159e 100644 --- a/internal/constellation/constellation.go +++ b/internal/constellation/constellation.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/constellation/featureset/featureset.go b/internal/constellation/featureset/featureset.go index b2ebb9cf7..c20f9c361 100644 --- a/internal/constellation/featureset/featureset.go +++ b/internal/constellation/featureset/featureset.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // package featureset provides a way to check whether a feature is enabled in the current build. diff --git a/internal/constellation/featureset/featureset_enterprise.go b/internal/constellation/featureset/featureset_enterprise.go index 3cd69c785..d79ccc006 100644 --- a/internal/constellation/featureset/featureset_enterprise.go +++ b/internal/constellation/featureset/featureset_enterprise.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package featureset diff --git a/internal/constellation/featureset/featureset_oss.go b/internal/constellation/featureset/featureset_oss.go index 2072641d3..82f95317d 100644 --- a/internal/constellation/featureset/featureset_oss.go +++ b/internal/constellation/featureset/featureset_oss.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package featureset diff --git a/internal/constellation/helm.go b/internal/constellation/helm.go index e8b9a815f..7d9cca7de 100644 --- a/internal/constellation/helm.go +++ b/internal/constellation/helm.go @@ -1,34 +1,94 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation import ( + "context" "errors" + "fmt" - "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constellation/helm" "github.com/edgelesssys/constellation/v2/internal/constellation/state" "github.com/edgelesssys/constellation/v2/internal/kms/uri" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" ) +var patch = []byte(fmt.Sprintf(`{"metadata": {"labels": {%q: %q}, "annotations": {%q: %q, %q: %q}}}`, + "app.kubernetes.io/managed-by", "Helm", + "meta.helm.sh/release-name", "coredns", + "meta.helm.sh/release-namespace", "kube-system")) + +var namespacedCoreDNSResources = map[schema.GroupVersionResource]string{ + {Group: "apps", Version: "v1", Resource: "deployments"}: "coredns", + {Group: "", Version: "v1", Resource: "services"}: "kube-dns", + {Group: "", Version: "v1", Resource: "configmaps"}: "coredns", + {Group: "", Version: "v1", Resource: "serviceaccounts"}: "coredns", + {Group: "apps", Version: "v1", Resource: "statefulsets"}: "foobarbax", +} + +var coreDNSResources = map[schema.GroupVersionResource]string{ + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}: "system:coredns", + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}: "system:coredns", +} + +// AnnotateCoreDNSResources imports existing CoreDNS resources into the Helm release. +// +// This is only required when CoreDNS was installed by kubeadm directly. +// TODO(burgerdev): remove after v2.19 is released. +func (a *Applier) AnnotateCoreDNSResources(ctx context.Context) error { + for gvk, name := range coreDNSResources { + _, err := a.dynamicClient.Resource(gvk).Patch(ctx, name, types.StrategicMergePatchType, patch, v1.PatchOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + for gvk, name := range namespacedCoreDNSResources { + _, err := a.dynamicClient.Resource(gvk).Namespace("kube-system").Patch(ctx, name, types.StrategicMergePatchType, patch, v1.PatchOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + return nil +} + +// CleanupCoreDNSResources removes CoreDNS resources that are not managed by Helm. +// +// This is only required when CoreDNS was installed by kubeadm directly. +// TODO(burgerdev): remove after v2.19 is released. +func (a *Applier) CleanupCoreDNSResources(ctx context.Context) error { + err := a.dynamicClient. + Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}). + Namespace("kube-system"). + Delete(ctx, "coredns", v1.DeleteOptions{}) + if !k8serrors.IsNotFound(err) { + return err + } + return nil +} + // PrepareHelmCharts loads Helm charts for Constellation and returns an executor to apply them. func (a *Applier) PrepareHelmCharts( - flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig, + flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) { if a.helmClient == nil { return nil, false, errors.New("helm client not initialized") } - return a.helmClient.PrepareApply(flags, state, serviceAccURI, masterSecret, openStackCfg) + return a.helmClient.PrepareApply(flags, state, serviceAccURI, masterSecret) } type helmApplier interface { PrepareApply( - flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig, + flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) ( helm.Applier, bool, error) } diff --git a/internal/constellation/helm/BUILD.bazel b/internal/constellation/helm/BUILD.bazel index 559d89358..a36717668 100644 --- a/internal/constellation/helm/BUILD.bazel +++ b/internal/constellation/helm/BUILD.bazel @@ -17,19 +17,37 @@ go_library( "versionlister.go", ], embedsrcs = [ + "charts/aws-load-balancer-controller/.helmignore", + "charts/aws-load-balancer-controller/Chart.yaml", + "charts/aws-load-balancer-controller/crds/crds.yaml", + "charts/aws-load-balancer-controller/README.md", + "charts/aws-load-balancer-controller/templates/_helpers.tpl", + "charts/aws-load-balancer-controller/templates/deployment.yaml", + "charts/aws-load-balancer-controller/templates/ingressclass.yaml", + "charts/aws-load-balancer-controller/templates/NOTES.txt", + "charts/aws-load-balancer-controller/templates/pdb.yaml", + "charts/aws-load-balancer-controller/templates/rbac.yaml", + "charts/aws-load-balancer-controller/templates/service.yaml", + "charts/aws-load-balancer-controller/templates/serviceaccount.yaml", + "charts/aws-load-balancer-controller/templates/servicemonitor.yaml", + "charts/aws-load-balancer-controller/templates/webhook.yaml", + "charts/aws-load-balancer-controller/values.yaml", "charts/cert-manager/Chart.yaml", - "charts/cert-manager/templates/NOTES.txt", "charts/cert-manager/templates/_helpers.tpl", "charts/cert-manager/templates/cainjector-deployment.yaml", + "charts/cert-manager/templates/cainjector-poddisruptionbudget.yaml", "charts/cert-manager/templates/cainjector-psp-clusterrole.yaml", "charts/cert-manager/templates/cainjector-psp-clusterrolebinding.yaml", "charts/cert-manager/templates/cainjector-psp.yaml", "charts/cert-manager/templates/cainjector-rbac.yaml", "charts/cert-manager/templates/cainjector-serviceaccount.yaml", + "charts/cert-manager/templates/controller-config.yaml", "charts/cert-manager/templates/crds.yaml", "charts/cert-manager/templates/deployment.yaml", "charts/cert-manager/templates/networkpolicy-egress.yaml", "charts/cert-manager/templates/networkpolicy-webhooks.yaml", + "charts/cert-manager/templates/NOTES.txt", + "charts/cert-manager/templates/poddisruptionbudget.yaml", "charts/cert-manager/templates/psp-clusterrole.yaml", "charts/cert-manager/templates/psp-clusterrolebinding.yaml", "charts/cert-manager/templates/psp.yaml", @@ -46,6 +64,7 @@ go_library( "charts/cert-manager/templates/webhook-config.yaml", "charts/cert-manager/templates/webhook-deployment.yaml", "charts/cert-manager/templates/webhook-mutating-webhook.yaml", + "charts/cert-manager/templates/webhook-poddisruptionbudget.yaml", "charts/cert-manager/templates/webhook-psp-clusterrole.yaml", "charts/cert-manager/templates/webhook-psp-clusterrolebinding.yaml", "charts/cert-manager/templates/webhook-psp.yaml", @@ -56,29 +75,50 @@ go_library( "charts/cert-manager/values.yaml", "charts/cilium/.helmignore", "charts/cilium/Chart.yaml", - "charts/cilium/LICENSE", - "charts/cilium/README.md", - "charts/cilium/README.md.gotmpl", + "charts/cilium/files/agent/poststart-eni.bash", + "charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json", + "charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json", + "charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json", + "charts/cilium/files/hubble/dashboards/hubble-dashboard.json", + "charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json", + "charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json", + "charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json", "charts/cilium/files/nodeinit/poststart-eni.bash", "charts/cilium/files/nodeinit/prestop.bash", "charts/cilium/files/nodeinit/startup.bash", - "charts/cilium/templates/NOTES.txt", + "charts/cilium/files/spire/init.bash", + "charts/cilium/files/spire/wait-for-spire.bash", + "charts/cilium/LICENSE", + "charts/cilium/README.md.gotmpl", + "charts/cilium/README.md", "charts/cilium/templates/_helpers.tpl", "charts/cilium/templates/cilium-agent/clusterrole.yaml", "charts/cilium/templates/cilium-agent/clusterrolebinding.yaml", "charts/cilium/templates/cilium-agent/daemonset.yaml", + "charts/cilium/templates/cilium-agent/dashboards-configmap.yaml", "charts/cilium/templates/cilium-agent/role.yaml", "charts/cilium/templates/cilium-agent/rolebinding.yaml", "charts/cilium/templates/cilium-agent/service.yaml", "charts/cilium/templates/cilium-agent/serviceaccount.yaml", "charts/cilium/templates/cilium-agent/servicemonitor.yaml", + "charts/cilium/templates/cilium-ca-bundle-configmap.yaml", "charts/cilium/templates/cilium-ca-secret.yaml", "charts/cilium/templates/cilium-configmap.yaml", + "charts/cilium/templates/cilium-envoy/configmap.yaml", + "charts/cilium/templates/cilium-envoy/daemonset.yaml", + "charts/cilium/templates/cilium-envoy/service.yaml", + "charts/cilium/templates/cilium-envoy/serviceaccount.yaml", + "charts/cilium/templates/cilium-envoy/servicemonitor.yaml", + "charts/cilium/templates/cilium-flowlog-configmap.yaml", + "charts/cilium/templates/cilium-gateway-api-class.yaml", "charts/cilium/templates/cilium-ingress-class.yaml", + "charts/cilium/templates/cilium-ingress-service.yaml", "charts/cilium/templates/cilium-nodeinit/daemonset.yaml", + "charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml", "charts/cilium/templates/cilium-operator/_helpers.tpl", "charts/cilium/templates/cilium-operator/clusterrole.yaml", "charts/cilium/templates/cilium-operator/clusterrolebinding.yaml", + "charts/cilium/templates/cilium-operator/dashboards-configmap.yaml", "charts/cilium/templates/cilium-operator/deployment.yaml", "charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml", "charts/cilium/templates/cilium-operator/role.yaml", @@ -95,12 +135,15 @@ go_library( "charts/cilium/templates/cilium-preflight/serviceaccount.yaml", "charts/cilium/templates/cilium-resource-quota.yaml", "charts/cilium/templates/cilium-secrets-namespace.yaml", + "charts/cilium/templates/clustermesh-apiserver/_helpers.tpl", "charts/cilium/templates/clustermesh-apiserver/clusterrole.yaml", "charts/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml", "charts/cilium/templates/clustermesh-apiserver/deployment.yaml", + "charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml", "charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml", "charts/cilium/templates/clustermesh-apiserver/service.yaml", "charts/cilium/templates/clustermesh-apiserver/serviceaccount.yaml", + "charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml", "charts/cilium/templates/clustermesh-apiserver/tls-certmanager/_helpers.tpl", "charts/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml", "charts/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml", @@ -125,8 +168,10 @@ go_library( "charts/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml", "charts/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml", "charts/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml", + "charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml", "charts/cilium/templates/clustermesh-config/_helpers.tpl", "charts/cilium/templates/clustermesh-config/clustermesh-secret.yaml", + "charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml", "charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml", "charts/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml", "charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml", @@ -151,6 +196,7 @@ go_library( "charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml", "charts/cilium/templates/hubble-ui/service.yaml", "charts/cilium/templates/hubble-ui/serviceaccount.yaml", + "charts/cilium/templates/hubble/dashboards-configmap.yaml", "charts/cilium/templates/hubble/metrics-service.yaml", "charts/cilium/templates/hubble/peer-service.yaml", "charts/cilium/templates/hubble/servicemonitor.yaml", @@ -178,9 +224,25 @@ go_library( "charts/cilium/templates/hubble/tls-provided/relay-server-secret.yaml", "charts/cilium/templates/hubble/tls-provided/server-secret.yaml", "charts/cilium/templates/hubble/tls-provided/ui-client-certs.yaml", + "charts/cilium/templates/NOTES.txt", + "charts/cilium/templates/spire/agent/clusterrole.yaml", + "charts/cilium/templates/spire/agent/clusterrolebinding.yaml", + "charts/cilium/templates/spire/agent/configmap.yaml", + "charts/cilium/templates/spire/agent/daemonset.yaml", + "charts/cilium/templates/spire/agent/serviceaccount.yaml", + "charts/cilium/templates/spire/bundle-configmap.yaml", + "charts/cilium/templates/spire/namespace.yaml", + "charts/cilium/templates/spire/server/clusterrole.yaml", + "charts/cilium/templates/spire/server/clusterrolebinding.yaml", + "charts/cilium/templates/spire/server/configmap.yaml", + "charts/cilium/templates/spire/server/role.yaml", + "charts/cilium/templates/spire/server/rolebinding.yaml", + "charts/cilium/templates/spire/server/service.yaml", + "charts/cilium/templates/spire/server/serviceaccount.yaml", + "charts/cilium/templates/spire/server/statefulset.yaml", "charts/cilium/templates/validate.yaml", - "charts/cilium/values.yaml", "charts/cilium/values.yaml.tmpl", + "charts/cilium/values.yaml", "charts/edgeless/constellation-services/.helmignore", "charts/edgeless/constellation-services/Chart.yaml", "charts/edgeless/constellation-services/charts/autoscaler/.helmignore", @@ -189,6 +251,7 @@ go_library( "charts/edgeless/constellation-services/charts/autoscaler/templates/azure-deployment.yaml", "charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrole.yaml", "charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml", + "charts/edgeless/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml", "charts/edgeless/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml", "charts/edgeless/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml", "charts/edgeless/constellation-services/charts/autoscaler/templates/role.yaml", @@ -203,9 +266,12 @@ go_library( "charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml", "charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml", "charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml", + "charts/edgeless/constellation-services/charts/ccm/templates/gcp-clusterrolebinding.yaml", "charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml", "charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml", "charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml", + "charts/edgeless/constellation-services/charts/ccm/templates/openstack-daemonset.yaml", + "charts/edgeless/constellation-services/charts/ccm/templates/openstack-secret.yaml", "charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml", "charts/edgeless/constellation-services/charts/ccm/values.schema.json", "charts/edgeless/constellation-services/charts/ccm/values.yaml", @@ -241,13 +307,6 @@ go_library( "charts/edgeless/constellation-services/charts/key-service/templates/serviceaccount.yaml", "charts/edgeless/constellation-services/charts/key-service/values.schema.json", "charts/edgeless/constellation-services/charts/key-service/values.yaml", - "charts/edgeless/constellation-services/charts/konnectivity/.helmignore", - "charts/edgeless/constellation-services/charts/konnectivity/Chart.yaml", - "charts/edgeless/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml", - "charts/edgeless/constellation-services/charts/konnectivity/templates/daemonset.yaml", - "charts/edgeless/constellation-services/charts/konnectivity/templates/serviceaccount.yaml", - "charts/edgeless/constellation-services/charts/konnectivity/values.schema.json", - "charts/edgeless/constellation-services/charts/konnectivity/values.yaml", "charts/edgeless/constellation-services/charts/verification-service/.helmignore", "charts/edgeless/constellation-services/charts/verification-service/Chart.yaml", "charts/edgeless/constellation-services/charts/verification-service/templates/daemonset.yaml", @@ -256,6 +315,88 @@ go_library( "charts/edgeless/constellation-services/charts/verification-service/values.yaml", "charts/edgeless/constellation-services/templates/.gitkeep", "charts/edgeless/constellation-services/values.yaml", + "charts/edgeless/csi/Chart.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md", + "charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/_helpers.tpl", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-attacher.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-csi-node.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-provisioner.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-resizer.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-snapshotter.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/csidriver.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt", + "charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/storageclass_default.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/storageclass_integrity.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/storageclass.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/values.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/Chart.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/_helpers.tpl", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-driver.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-controller.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-node.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/storageclass_default.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/storageclass_integrity.yaml", + "charts/edgeless/csi/charts/azuredisk-csi-driver/values.yaml", + "charts/edgeless/csi/charts/cinder-config/.helmignore", + "charts/edgeless/csi/charts/cinder-config/Chart.yaml", + "charts/edgeless/csi/charts/cinder-config/templates/secret.yaml", + "charts/edgeless/csi/charts/cinder-config/values.schema.json", + "charts/edgeless/csi/charts/cinder-config/values.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/Chart.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/cluster_setup.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/controller.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/node.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/storageclass_default.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/storageclass_integrity.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/v1_csidriver.yaml", + "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/values.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/Chart.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/README.md", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/_helpers.tpl", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/cinder-csi-driver.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-deployment.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-podmonitor.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-rbac.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/custom_storageclass.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-daemonset.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-rbac.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/NOTES.txt", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/secret.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/templates/storageclass.yaml", + "charts/edgeless/csi/charts/openstack-cinder-csi/values.yaml", + "charts/edgeless/csi/charts/snapshot-controller/Chart.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/admission-configuration.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/rbac-snapshot-controller.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/rbac-snapshot-webhook.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/selfsigned-issuer.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/serving-cert.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-controller.yaml", + "charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-webhook.yaml", + "charts/edgeless/csi/charts/snapshot-controller/values.yaml", + "charts/edgeless/csi/charts/snapshot-crds/Chart.yaml", + "charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotclasses.yaml", + "charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotcontents.yaml", + "charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshots.yaml", + "charts/edgeless/csi/charts/snapshot-crds/values.yaml", + "charts/edgeless/csi/values.yaml", "charts/edgeless/operators/.helmignore", "charts/edgeless/operators/Chart.yaml", "charts/edgeless/operators/charts/constellation-operator/.helmignore", @@ -292,171 +433,48 @@ go_library( "charts/edgeless/operators/charts/node-maintenance-operator/values.schema.json", "charts/edgeless/operators/charts/node-maintenance-operator/values.yaml", "charts/edgeless/operators/values.yaml", - "charts/edgeless/constellation-services/charts/ccm/templates/openstack-daemonset.yaml", - "charts/edgeless/constellation-services/charts/ccm/templates/openstack-secret.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/Chart.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/README.md", - "charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/_helpers.tpl", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/rbac-yawol-cloud-controller.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/rbac-yawol-controller.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/sa-yawol-cloud-controller.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/sa-yawol-controller.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/vpa.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-cloud-controller.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-controller.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml", - "charts/edgeless/constellation-services/charts/yawol-controller/values.yaml", - "charts/edgeless/constellation-services/charts/yawol-config/.helmignore", - "charts/edgeless/constellation-services/charts/yawol-config/Chart.yaml", - "charts/edgeless/constellation-services/charts/yawol-config/templates/secret.yaml", - "charts/edgeless/constellation-services/charts/yawol-config/values.schema.json", - "charts/edgeless/constellation-services/charts/yawol-config/values.yaml", - "charts/aws-load-balancer-controller/.helmignore", - "charts/aws-load-balancer-controller/Chart.yaml", - "charts/aws-load-balancer-controller/README.md", - "charts/aws-load-balancer-controller/crds/crds.yaml", - "charts/aws-load-balancer-controller/templates/NOTES.txt", - "charts/aws-load-balancer-controller/templates/_helpers.tpl", - "charts/aws-load-balancer-controller/templates/deployment.yaml", - "charts/aws-load-balancer-controller/templates/ingressclass.yaml", - "charts/aws-load-balancer-controller/templates/pdb.yaml", - "charts/aws-load-balancer-controller/templates/rbac.yaml", - "charts/aws-load-balancer-controller/templates/service.yaml", - "charts/aws-load-balancer-controller/templates/serviceaccount.yaml", - "charts/aws-load-balancer-controller/templates/servicemonitor.yaml", - "charts/aws-load-balancer-controller/templates/webhook.yaml", - "charts/aws-load-balancer-controller/values.yaml", - "charts/edgeless/csi/Chart.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/Chart.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/_helpers.tpl", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-driver.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-controller.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-node.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/storageclass_default.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/templates/storageclass_integrity.yaml", - "charts/edgeless/csi/charts/azuredisk-csi-driver/values.yaml", - "charts/edgeless/csi/charts/cinder-config/.helmignore", - "charts/edgeless/csi/charts/cinder-config/Chart.yaml", - "charts/edgeless/csi/charts/cinder-config/templates/secret.yaml", - "charts/edgeless/csi/charts/cinder-config/values.schema.json", - "charts/edgeless/csi/charts/cinder-config/values.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/Chart.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/cluster_setup.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/controller.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/node.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/storageclass_default.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/storageclass_integrity.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/templates/v1_csidriver.yaml", - "charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/values.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/Chart.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/README.md", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/NOTES.txt", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/_helpers.tpl", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/cinder-csi-driver.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-deployment.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-rbac.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/custom_storageclass.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-daemonset.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-rbac.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/secret.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/templates/storageclass.yaml", - "charts/edgeless/csi/charts/openstack-cinder-csi/values.yaml", - "charts/edgeless/csi/charts/snapshot-controller/Chart.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/admission-configuration.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/rbac-snapshot-controller.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/rbac-snapshot-webhook.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/selfsigned-issuer.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/serving-cert.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-controller.yaml", - "charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-webhook.yaml", - "charts/edgeless/csi/charts/snapshot-controller/values.yaml", - "charts/edgeless/csi/charts/snapshot-crds/Chart.yaml", - "charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotclasses.yaml", - "charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotcontents.yaml", - "charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshots.yaml", - "charts/edgeless/csi/charts/snapshot-crds/values.yaml", - "charts/edgeless/csi/values.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md", - "charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt", - "charts/edgeless/csi/charts/aws-csi-driver/templates/_helpers.tpl", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-attacher.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-csi-node.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-provisioner.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-resizer.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrolebinding-snapshotter.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/csidriver.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/storageclass.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/storageclass_default.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/storageclass_integrity.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml", - "charts/edgeless/csi/charts/aws-csi-driver/values.yaml", - "charts/edgeless/constellation-services/charts/ccm/templates/gcp-clusterrolebinding.yaml", - "charts/cilium/files/agent/poststart-eni.bash", - "charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json", - "charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json", - "charts/cilium/files/cilium-operator/dashboards/cilium-operator-dashboard.json", - "charts/cilium/files/hubble/dashboards/hubble-dashboard.json", - "charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json", - "charts/cilium/files/hubble/dashboards/hubble-l7-http-metrics-by-workload.json", - "charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json", - "charts/cilium/files/spire/init.bash", - "charts/cilium/files/spire/wait-for-spire.bash", - "charts/cilium/templates/cilium-agent/dashboards-configmap.yaml", - "charts/cilium/templates/cilium-ca-bundle-configmap.yaml", - "charts/cilium/templates/cilium-envoy/configmap.yaml", - "charts/cilium/templates/cilium-envoy/daemonset.yaml", - "charts/cilium/templates/cilium-envoy/service.yaml", - "charts/cilium/templates/cilium-envoy/serviceaccount.yaml", - "charts/cilium/templates/cilium-envoy/servicemonitor.yaml", - "charts/cilium/templates/cilium-gateway-api-class.yaml", - "charts/cilium/templates/cilium-ingress-service.yaml", - "charts/cilium/templates/cilium-nodeinit/serviceaccount.yaml", - "charts/cilium/templates/cilium-operator/dashboards-configmap.yaml", - "charts/cilium/templates/clustermesh-apiserver/_helpers.tpl", - "charts/cilium/templates/clustermesh-apiserver/metrics-service.yaml", - "charts/cilium/templates/clustermesh-apiserver/servicemonitor.yaml", - "charts/cilium/templates/clustermesh-apiserver/users-configmap.yaml", - "charts/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml", - "charts/cilium/templates/hubble/dashboards-configmap.yaml", - "charts/cilium/templates/spire/agent/clusterrole.yaml", - "charts/cilium/templates/spire/agent/clusterrolebinding.yaml", - "charts/cilium/templates/spire/agent/configmap.yaml", - "charts/cilium/templates/spire/agent/daemonset.yaml", - "charts/cilium/templates/spire/agent/serviceaccount.yaml", - "charts/cilium/templates/spire/bundle-configmap.yaml", - "charts/cilium/templates/spire/namespace.yaml", - "charts/cilium/templates/spire/server/clusterrole.yaml", - "charts/cilium/templates/spire/server/clusterrolebinding.yaml", - "charts/cilium/templates/spire/server/configmap.yaml", - "charts/cilium/templates/spire/server/role.yaml", - "charts/cilium/templates/spire/server/rolebinding.yaml", - "charts/cilium/templates/spire/server/service.yaml", - "charts/cilium/templates/spire/server/serviceaccount.yaml", - "charts/cilium/templates/spire/server/statefulset.yaml", - "charts/cert-manager/templates/cainjector-poddisruptionbudget.yaml", - "charts/cert-manager/templates/controller-config.yaml", - "charts/cert-manager/templates/poddisruptionbudget.yaml", - "charts/cert-manager/templates/webhook-poddisruptionbudget.yaml", + "charts/yawol/.helmignore", + "charts/yawol/Chart.yaml", + "charts/yawol/charts/yawol-config/.helmignore", + "charts/yawol/charts/yawol-config/Chart.yaml", + "charts/yawol/charts/yawol-config/templates/secret.yaml", + "charts/yawol/charts/yawol-config/values.schema.json", + "charts/yawol/charts/yawol-config/values.yaml", + "charts/yawol/charts/yawol-controller/Chart.yaml", + "charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml", + "charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml", + "charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml", + "charts/yawol/charts/yawol-controller/README.md", + "charts/yawol/charts/yawol-controller/templates/_helpers.tpl", + "charts/yawol/charts/yawol-controller/templates/rbac-yawol-cloud-controller.yaml", + "charts/yawol/charts/yawol-controller/templates/rbac-yawol-controller.yaml", + "charts/yawol/charts/yawol-controller/templates/sa-yawol-cloud-controller.yaml", + "charts/yawol/charts/yawol-controller/templates/sa-yawol-controller.yaml", + "charts/yawol/charts/yawol-controller/templates/vpa.yaml", + "charts/yawol/charts/yawol-controller/templates/yawol-cloud-controller.yaml", + "charts/yawol/charts/yawol-controller/templates/yawol-controller.yaml", + "charts/yawol/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml", + "charts/yawol/charts/yawol-controller/values.yaml", + "charts/yawol/templates/.gitkeep", + "charts/yawol/values.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl", + "charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl", + "charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml", + "charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml", + "charts/cert-manager/templates/cainjector-config.yaml", + "charts/cert-manager/templates/extras-objects.yaml", + "charts/cert-manager/templates/podmonitor.yaml", + "charts/coredns/Chart.yaml", + "charts/coredns/values.yaml", + "charts/coredns/templates/clusterrole.yaml", + "charts/coredns/templates/clusterrolebinding.yaml", + "charts/coredns/templates/configmap.yaml", + "charts/coredns/templates/deployment.yaml", + "charts/coredns/templates/service.yaml", + "charts/coredns/templates/serviceaccount.yaml", + "charts/aws-load-balancer-controller/templates/hpa.yaml", + "charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml", ], importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/helm", visibility = ["//:__subpackages__"], @@ -467,7 +485,6 @@ go_library( "//internal/cloud/gcpshared", "//internal/cloud/openstack", "//internal/compatibility", - "//internal/config", "//internal/constants", "//internal/constellation/helm/imageversion", "//internal/constellation/state", @@ -485,11 +502,12 @@ go_library( "@io_k8s_client_go//restmapper", "@io_k8s_client_go//tools/clientcmd", "@io_k8s_client_go//util/retry", - "@sh_helm_helm//pkg/ignore", + "@io_k8s_kubernetes//cmd/kubeadm/app/constants", "@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/chart", "@sh_helm_helm_v3//pkg/chart/loader", "@sh_helm_helm_v3//pkg/chartutil", + "@sh_helm_helm_v3//pkg/ignore", "@sh_helm_helm_v3//pkg/release", ], ) @@ -510,6 +528,7 @@ go_test( "//internal/cloud/azureshared", "//internal/cloud/cloudprovider", "//internal/cloud/gcpshared", + "//internal/cloud/openstack", "//internal/compatibility", "//internal/config", "//internal/constellation/state", diff --git a/internal/constellation/helm/action.go b/internal/constellation/helm/action.go index b2ca33242..f405beb29 100644 --- a/internal/constellation/helm/action.go +++ b/internal/constellation/helm/action.go @@ -1,15 +1,17 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm import ( "context" + "errors" "fmt" "path/filepath" + "strings" "time" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -37,7 +39,7 @@ type applyAction interface { func newActionConfig(kubeConfig []byte, logger debugLog) (*action.Configuration, error) { actionConfig := &action.Configuration{} if err := actionConfig.Init(&clientGetter{kubeConfig: kubeConfig}, constants.HelmNamespace, - "secret", logger.Debugf); err != nil { + "secret", helmLog(logger)); err != nil { return nil, err } return actionConfig, nil @@ -52,6 +54,12 @@ func newHelmInstallAction(config *action.Configuration, release release, timeout return action } +func newHelmUninstallAction(config *action.Configuration, timeout time.Duration) *action.Uninstall { + action := action.NewUninstall(config) + action.Timeout = timeout + return action +} + func setWaitMode(a *action.Install, waitMode WaitMode) { switch waitMode { case WaitModeNone: @@ -70,11 +78,12 @@ func setWaitMode(a *action.Install, waitMode WaitMode) { // installAction is an action that installs a helm chart. type installAction struct { - preInstall func(context.Context) error - release release - helmAction *action.Install - postInstall func(context.Context) error - log debugLog + preInstall func(context.Context) error + release release + helmAction *action.Install + uninstallAction *action.Uninstall + postInstall func(context.Context) error + log debugLog } // Apply installs the chart. @@ -103,6 +112,11 @@ func (a *installAction) SaveChart(chartsDir string, fileHandler file.Handler) er func (a *installAction) apply(ctx context.Context) error { _, err := a.helmAction.RunWithContext(ctx, a.release.chart, a.release.values) + if isUninstallError(err) && a.uninstallAction != nil { + a.log.Debug("cleaning up manually after failed atomic Helm install", "error", err, "release", a.release.releaseName) + _, uninstallErr := a.uninstallAction.Run(a.release.releaseName) + err = errors.Join(err, uninstallErr) + } return err } @@ -214,7 +228,7 @@ func (c *clientGetter) ToRESTMapper() (meta.RESTMapper, error) { return nil, err } mapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) - expander := restmapper.NewShortcutExpander(mapper, discoveryClient) + expander := restmapper.NewShortcutExpander(mapper, discoveryClient, nil) return expander, nil } @@ -222,3 +236,14 @@ func (c *clientGetter) ToRESTMapper() (meta.RESTMapper, error) { func (c *clientGetter) ToRawKubeConfigLoader() clientcmd.ClientConfig { return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{}, &clientcmd.ConfigOverrides{}) } + +func helmLog(log debugLog) action.DebugLog { + return func(format string, v ...interface{}) { + log.Debug(fmt.Sprintf(format, v...)) + } +} + +func isUninstallError(err error) bool { + return err != nil && (strings.Contains(err.Error(), "an error occurred while uninstalling the release") || + strings.Contains(err.Error(), "cannot re-use a name that is still in use")) +} diff --git a/internal/constellation/helm/actionfactory.go b/internal/constellation/helm/actionfactory.go index 07425e7f6..059ebe712 100644 --- a/internal/constellation/helm/actionfactory.go +++ b/internal/constellation/helm/actionfactory.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm @@ -90,15 +90,15 @@ func (a actionFactory) appendNewAction( ) } - a.log.Debugf("release %s not found, adding to new releases...", release.releaseName) + a.log.Debug(fmt.Sprintf("release %q not found, adding to new releases...", release.releaseName)) *actions = append(*actions, a.newInstall(release, timeout)) return nil } if err != nil { return fmt.Errorf("getting version for %s: %w", release.releaseName, err) } - a.log.Debugf("Current %s version: %s", release.releaseName, currentVersion) - a.log.Debugf("New %s version: %s", release.releaseName, newVersion) + a.log.Debug(fmt.Sprintf("Current %q version: %q", release.releaseName, currentVersion)) + a.log.Debug(fmt.Sprintf("New %q version: %q", release.releaseName, newVersion)) if !force { // For charts we package ourselves, the version is equal to the CLI version (charts are embedded in the binary). @@ -120,8 +120,7 @@ func (a actionFactory) appendNewAction( } else { // This may break for external chart dependencies if we decide to upgrade more than one minor version at a time. if err := newVersion.IsUpgradeTo(currentVersion); err != nil { - // TODO(3u13r): Remove when Constellation v2.14 is released. - // We need to ignore that we jump from Cilium v1.12 to v1.15-pre. We have verified that this works. + // Allow bigger Cilium and Cert-Manager version jumps. if !(errors.Is(err, compatibility.ErrMinorDrift) && (release.releaseName == "cilium" || release.releaseName == "cert-manager")) { return fmt.Errorf("invalid upgrade for %s: %w", release.releaseName, err) } @@ -133,13 +132,16 @@ func (a actionFactory) appendNewAction( release.releaseName == certManagerInfo.releaseName { return ErrConfirmationMissing } - a.log.Debugf("Upgrading %s from %s to %s", release.releaseName, currentVersion, newVersion) + a.log.Debug(fmt.Sprintf("Upgrading %q from %q to %q", release.releaseName, currentVersion, newVersion)) *actions = append(*actions, a.newUpgrade(release, timeout)) return nil } func (a actionFactory) newInstall(release release, timeout time.Duration) *installAction { action := &installAction{helmAction: newHelmInstallAction(a.cfg, release, timeout), release: release, log: a.log} + if action.IsAtomic() { + action.uninstallAction = newHelmUninstallAction(a.cfg, timeout) + } return action } @@ -163,7 +165,7 @@ func (a actionFactory) updateCRDs(ctx context.Context, chart *chart.Chart) error for _, dep := range chart.Dependencies() { for _, crdFile := range dep.Files { if strings.HasPrefix(crdFile.Name, "crds/") { - a.log.Debugf("Updating crd: %s", crdFile.Name) + a.log.Debug(fmt.Sprintf("Updating crd: %q", crdFile.Name)) err := a.kubeClient.ApplyCRD(ctx, crdFile.Data) if err != nil { return err diff --git a/internal/constellation/helm/actionfactory_test.go b/internal/constellation/helm/actionfactory_test.go index 960ea5a52..93ec54dc8 100644 --- a/internal/constellation/helm/actionfactory_test.go +++ b/internal/constellation/helm/actionfactory_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/Chart.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/Chart.yaml index 363fff854..1f1b4e9ba 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/Chart.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: aws-load-balancer-controller description: AWS Load Balancer Controller Helm chart for Kubernetes -version: 1.5.4 -appVersion: v2.5.3 +version: 1.11.0 +appVersion: v2.11.0 home: https://github.com/aws/eks-charts icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png sources: diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/README.md b/internal/constellation/helm/charts/aws-load-balancer-controller/README.md index ee4be9aad..180e50c08 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/README.md +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/README.md @@ -22,7 +22,11 @@ AWS Load Balancer controller manages the following AWS resources As a security best practice, we recommend isolating the controller deployment pods to specific node groups which run critical components. The helm chart provides parameters ```nodeSelector```, ```tolerations``` and ```affinity``` to configure node isolation. For more information, please refer to the guidance [here](https://aws.github.io/aws-eks-best-practices/security/docs/multitenancy/#isolating-tenant-workloads-to-specific-nodes). ## Prerequisites -- Kubernetes >= 1.19 +- Supported Kubernetes Versions + - Chart version v1.5.0+ requires Kubernetes 1.22+ + - Chart version v1.4.0+ requires Kubernetes 1.19+ + - Chart version v1.2.0 - v1.3.3 supports Kubernetes 1.16-1.21 + - Chart version v1.1.6 and before supports Kubernetes 1.15 - IAM permissions - Helm v3 - Optional dependencies @@ -74,7 +78,7 @@ If migrating from ALB ingress controller, grant [additional IAM permissions](htt - Additional IAM permissions required, ensure you have granted the [required IAM permissions](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json). - CRDs need to be updated as follows ```shell script -kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" +kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master" ``` - you can run helm upgrade without uninstalling the old chart completely @@ -92,8 +96,11 @@ If you are setting `serviceMonitor.enabled: true` you need to have installed the ## Installing the Chart **Note**: You need to uninstall aws-alb-ingress-controller. Please refer to the [upgrade](#Upgrade) section below before you proceed. + **Note**: Starting chart version 1.4.1, you need to explicitly set `clusterSecretsPermissions.allowAllSecrets` to true to grant the controller permission to access all secrets for OIDC feature. We recommend configuring access to individual secrets resource separately [[link](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/examples/secrets_access/)]. +**Note**: To ensure compatibility, we recommend installing the AWS Load Balancer controller image version with its compatible Helm chart version. Use the ```helm search repo eks/aws-load-balancer-controller --versions``` command to find the compatible versions. + Add the EKS repository to Helm: ```shell script helm repo add eks https://aws.github.io/eks-charts @@ -102,7 +109,7 @@ helm repo add eks https://aws.github.io/eks-charts Install the TargetGroupBinding CRDs: ```shell script -kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master" +kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master" ``` Install the AWS Load Balancer controller, if using iamserviceaccount @@ -171,88 +178,106 @@ Chart release v1.2.0 and later enables high availability configuration by defaul The following tables lists the configurable parameters of the chart and their default values. The default values set by the application itself can be confirmed [here](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/deploy/configurations/#controller-configuration-options). -| Parameter | Description | Default | -|------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------| -| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | -| `image.tag` | image tag | `` | -| `image.pullPolicy` | image pull policy | `IfNotPresent` | -| `clusterName` | Kubernetes cluster name | None | -| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | -| `securityContext` | Set to security context for pod | `{}` | -| `resources` | Controller pod resource requests & limits | `{}` | -| `priorityClassName` | Controller pod priority class | system-cluster-critical | -| `nodeSelector` | Node labels for controller pod assignment | `{}` | -| `tolerations` | Controller pod toleration for taints | `{}` | -| `affinity` | Affinity for pod assignment | `{}` | -| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | -| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | -| `deploymentAnnotations` | Annotations to add to deployment | `{}` | -| `podAnnotations` | Annotations to add to each pod | `{}` | -| `podLabels` | Labels to add to each pod | `{}` | -| `additionalLabels` | Labels to add to all components | `{}` | -| `rbac.create` | if `true`, create and use RBAC resources | `true` | -| `serviceAccount.annotations` | optional annotations to add to service account | None | -| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | -| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | -| `serviceAccount.create` | If `true`, create a new service account | `true` | -| `serviceAccount.name` | Service account to be used | None | -| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | -| `ingressClass` | The ingress class to satisfy | alb | -| `createIngressClassResource` | Create ingressClass resource | true | -| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | -| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | -| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | -| `region` | The AWS region for the kubernetes cluster | None | -| `vpcId` | The VPC ID for the Kubernetes cluster | None | -| `awsApiEndpoints` | Custom AWS API Endpoints | None | -| `awsApiThrottle` | Custom AWS API throttle settings | None | -| `awsMaxRetries` | Maximum retries for AWS APIs | None | -| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | -| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | -| `enableShield` | Enable Shield addon for ALB | None | -| `enableWaf` | Enable WAF addon for ALB | None | -| `enableWafv2` | Enable WAF V2 addon for ALB | None | -| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | -| `logLevel` | Set the controller log level - info, debug | None | -| `metricsBindAddr` | The address the metric endpoint binds to | "" | -| `webhookBindPort` | The TCP port the Webhook server binds to | None | -| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | -| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | -| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | -| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | -| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | -| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | -| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | -| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | -| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | -| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | -| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | -| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | -| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | -| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | -| `livenessProbe` | Liveness probe settings for the controller | (see `values.yaml`) | -| `env` | Environment variables to set for aws-load-balancer-controller pod | None | -| `hostNetwork` | If `true`, use hostNetwork | `false` | -| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | -| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | -| `extraVolumes` | Extra volumes for the pod | `[]` | -| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | -| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | -| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | -| `updateStrategy` | Defines the update strategy for the deployment | `{}` | -| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | -| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | -| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | -| `backendSecurityGroup` | Backend security group to use instead of auto created one if the feature is enabled | `` | -| `disableRestrictedSecurityGroupRules` | If disabled, controller will not specify port range restriction in the backend security group rules | `false` | -| `objectSelector.matchExpressions` | Webhook configuration to select specific pods by specifying the expression to be matched | None | -| `objectSelector.matchLabels` | Webhook configuration to select specific pods by specifying the key value label pair to be matched | None | -| `serviceMonitor.enabled` | Specifies whether a service monitor should be created, requires the ServiceMonitor CRD to be installed | `false` | -| `serviceMonitor.additionalLabels` | Labels to add to the service account | `{}` | -| `serviceMonitor.interval` | Prometheus scrape interval | `1m` | -| `serviceMonitor.namespace` | Namespace in which Prometheus is running | None | -| `clusterSecretsPermissions.allowAllSecrets` | If `true`, controller has access to all secrets in the cluster. | `false` | -| `controllerConfig.featureGates` | set of `key: value` pairs that describe AWS load balance controller features | `{}` | -| `ingressClassConfig.default` | If `true`, the ingressclass will be the default class of the cluster. | `false` | -| `enableServiceMutatorWebhook` | If `false`, disable the Service Mutator webhook which makes all new services of type LoadBalancer reconciled by the lb controller | `true` | + +| Parameter | Description | Default | +| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | +| `image.repository` | image repository | `public.ecr.aws/eks/aws-load-balancer-controller` | +| `image.tag` | image tag | `` | +| `image.pullPolicy` | image pull policy | `IfNotPresent` | +| `clusterName` | Kubernetes cluster name | None | +| `cluster.dnsDomain` | DNS domain of the Kubernetes cluster, included in TLS certificate requests | `cluster.local` | +| `securityContext` | Set to security context for pod | `{}` | +| `resources` | Controller pod resource requests & limits | `{}` | +| `priorityClassName` | Controller pod priority class | system-cluster-critical | +| `nodeSelector` | Node labels for controller pod assignment | `{}` | +| `tolerations` | Controller pod toleration for taints | `{}` | +| `affinity` | Affinity for pod assignment | `{}` | +| `configureDefaultAffinity` | Configure soft pod anti-affinity if custom affinity is not configured | `true` | +| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment | `{}` | +| `deploymentAnnotations` | Annotations to add to deployment | `{}` | +| `podAnnotations` | Annotations to add to each pod | `{}` | +| `podLabels` | Labels to add to each pod | `{}` | +| `additionalLabels` | Labels to add to all components | `{}` | +| `rbac.create` | if `true`, create and use RBAC resources | `true` | +| `serviceAccount.annotations` | optional annotations to add to service account | None | +| `serviceAccount.automountServiceAccountToken` | Automount API credentials for a Service Account | `true` | +| `serviceAccount.imagePullSecrets` | List of image pull secrets to add to the Service Account | `[]` | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used | None | +| `terminationGracePeriodSeconds` | Time period for controller pod to do a graceful shutdown | 10 | +| `ingressClass` | The ingress class to satisfy | alb | +| `createIngressClassResource` | Create ingressClass resource | true | +| `ingressClassParams.name` | IngressClassParams resource's name, default to the aws load balancer controller's name | None | +| `ingressClassParams.create` | If `true`, create a new ingressClassParams | true | +| `ingressClassParams.spec` | IngressClassParams defined ingress specifications | {} | +| `region` | The AWS region for the kubernetes cluster | None | +| `vpcId` | The VPC ID for the Kubernetes cluster | None | +| `awsApiEndpoints` | Custom AWS API Endpoints | None | +| `awsApiThrottle` | Custom AWS API throttle settings | None | +| `awsMaxRetries` | Maximum retries for AWS APIs | None | +| `defaultTargetType` | Default target type. Used as the default value of the `alb.ingress.kubernetes.io/target-type` and `service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" annotations.`Possible values are `ip` and `instance`. | `instance` | +| `enablePodReadinessGateInject` | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods | None | +| `enableShield` | Enable Shield addon for ALB | None | +| `enableWaf` | Enable WAF addon for ALB | None | +| `enableWafv2` | Enable WAF V2 addon for ALB | None | +| `ingressMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for ingress | None | +| `logLevel` | Set the controller log level - info, debug | None | +| `metricsBindAddr` | The address the metric endpoint binds to | "" | +| `webhookConfig.disableIngressValidation` | Disables the validation of resources of kind Ingress | None | +| `webhookBindPort` | The TCP port the Webhook server binds to | None | +| `webhookTLS.caCert` | TLS CA certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.cert` | TLS certificate for webhook (auto-generated if not provided) | "" | +| `webhookTLS.key` | TLS private key for webhook (auto-generated if not provided) | "" | +| `webhookNamespaceSelectors` | Namespace selectors for the wekbook | None | +| `keepTLSSecret` | Reuse existing TLS Secret during chart upgrade | `true` | +| `serviceAnnotations` | Annotations to be added to the provisioned webhook service resource | `{}` | +| `serviceMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for service | None | +| `targetgroupbindingMaxConcurrentReconciles` | Maximum number of concurrently running reconcile loops for targetGroupBinding | None | +| `targetgroupbindingMaxExponentialBackoffDelay` | Maximum duration of exponential backoff for targetGroupBinding reconcile failures | None | +| `syncPeriod` | Period at which the controller forces the repopulation of its local object stores | None | +| `watchNamespace` | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched | None | +| `disableIngressClassAnnotation` | Disables the usage of kubernetes.io/ingress.class annotation | None | +| `disableIngressGroupNameAnnotation` | Disables the usage of alb.ingress.kubernetes.io/group.name annotation | None | +| `tolerateNonExistentBackendService` | whether to allow rules that reference a backend service that does not exist. (When enabled, it will return 503 error if backend service not exist) | `true` | +| `tolerateNonExistentBackendAction` | whether to allow rules that reference a backend action that does not exist. (When enabled, it will return 503 error if backend action not exist) | `true` | +| `defaultSSLPolicy` | Specifies the default SSL policy to use for HTTPS or TLS listeners | None | +| `externalManagedTags` | Specifies the list of tag keys on AWS resources that are managed externally | `[]` | +| `livenessProbe` | Liveness probe settings for the controller | (see `values.yaml`) | +| `env` | Environment variables to set for aws-load-balancer-controller pod | None | +| `envFrom` | Environment variables to set for aws-load-balancer-controller pod from configMap or Secret | None | +| `envSecretName` | AWS credentials as environment variables from Secret (Secret keys `key_id` and `access_key`). | None | +| `hostNetwork` | If `true`, use hostNetwork | `false` | +| `dnsPolicy` | Set dnsPolicy if required | `ClusterFirst` | +| `extraVolumeMounts` | Extra volume mounts for the pod | `[]` | +| `extraVolumes` | Extra volumes for the pod | `[]` | +| `defaultTags` | Default tags to apply to all AWS resources managed by this controller | `{}` | +| `replicaCount` | Number of controller pods to run, only one will be active due to leader election | `2` | +| `revisionHistoryLimit` | Number of revisions to keep | `10` | +| `podDisruptionBudget` | Limit the disruption for controller pods. Require at least 2 controller replicas and 3 worker nodes | `{}` | +| `updateStrategy` | Defines the update strategy for the deployment | `{}` | +| `enableCertManager` | If enabled, cert-manager issues the webhook certificates instead of the helm template, requires cert-manager and it's CRDs to be installed | `false` | +| `enableEndpointSlices` | If enabled, controller uses k8s EndpointSlices instead of Endpoints for IP targets | `false` | +| `enableBackendSecurityGroup` | If enabled, controller uses shared security group for backend traffic | `true` | +| `backendSecurityGroup` | Backend security group to use instead of auto created one if the feature is enabled | `` | +| `disableRestrictedSecurityGroupRules` | If disabled, controller will not specify port range restriction in the backend security group rules | `false` | +| `objectSelector.matchExpressions` | Webhook configuration to select specific pods by specifying the expression to be matched | None | +| `objectSelector.matchLabels` | Webhook configuration to select specific pods by specifying the key value label pair to be matched | None | +| `serviceMonitor.enabled` | Specifies whether a service monitor should be created, requires the ServiceMonitor CRD to be installed | `false` | +| `serviceMonitor.namespace` | Namespace in which to create the service monitor | None | +| `serviceMonitor.additionalLabels` | Labels to add to the service monitor | `{}` | +| `serviceMonitor.interval` | Prometheus scrape interval | `1m` | +| `serviceMonitor.scrapeTimeout` | Prometheus scrape timeout | `1m` | +| `serviceMonitor.relabelings` | Relabelings to apply to samples before ingestion | `1m` | +| `serviceMonitor.metricRelabelings` | Metric relabelings to apply to samples before ingestion | `1m` | +| `clusterSecretsPermissions.allowAllSecrets` | If `true`, controller has access to all secrets in the cluster. | `false` | +| `controllerConfig.featureGates` | set of `key: value` pairs that describe AWS load balance controller features | `{}` | +| `ingressClassConfig.default` | If `true`, the ingressclass will be the default class of the cluster. | `false` | +| `enableServiceMutatorWebhook` | If `false`, disable the Service Mutator webhook which makes all new services of type LoadBalancer reconciled by the lb controller | `true` | +| `serviceMutatorWebhookConfig.failurePolicy` | Failure policy for the Service Mutator webhook | `Fail` | +| `serviceMutatorWebhookConfig.objectSelector` | Object selector(s) to limit which objects will be mutated by the Service Mutator webhook | `[]` | +| `serviceMutatorWebhookConfig.operations` | List of operations that will trigger the the Service Mutator webhook | `[ CREATE ]` | +| `autoscaling` | If `autoscaling.enabled=true`, enable the HPA on the controller mainly to survive load induced failure by the calls to the `aws-load-balancer-webhook-service`. Please keep in mind that the controller pods have `priorityClassName: system-cluster-critical`, enabling HPA may lead to the eviction of other low-priority pods in the node | `false` | +| `serviceTargetENISGTags` | set of `key=value` pairs of AWS tags in addition to cluster name for finding the target ENI security group to which to add inbound rules from NLBs | None | +| `loadBalancerClass` | Sets the AWS load balancer type to be used when the Kubernetes service requests an external load balancer | `service.k8s.aws/nlb` | +| `creator` | if set to a `value!=helm`, it will disable the addition of default helm labels | `helm` | +| `runtimeClassName` | Runtime class name for the controller pods , such as `gvisor` or `kata`. An unspecified `nil` or empty `""` RuntimeClassName is equivalent to the backwards-compatible default behavior as if the RuntimeClass feature is disabled. | "" | diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/crds/crds.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/crds/crds.yaml index 78c226660..b72e68789 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/crds/crds.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/crds/crds.yaml @@ -2,8 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: ingressclassparams.elbv2.k8s.aws spec: group: elbv2.k8s.aws @@ -36,20 +35,31 @@ spec: description: IngressClassParams is the Schema for the IngressClassParams API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: IngressClassParamsSpec defines the desired state of IngressClassParams properties: + certificateArn: + description: CertificateArn specifies the ARN of the certificates + for all Ingresses that belong to IngressClass with this IngressClassParams. + items: + type: string + type: array group: description: Group defines the IngressGroup for all Ingresses that belong to IngressClass with this IngressClassParams. @@ -72,7 +82,38 @@ spec: enum: - ipv4 - dualstack + - dualstack-without-public-ipv4 type: string + listeners: + description: Listeners define a list of listeners with their protocol, + port and attributes. + items: + properties: + listenerAttributes: + description: The attributes of the listener + items: + description: Attributes defines custom attributes on resources. + properties: + key: + description: The key of the attribute. + type: string + value: + description: The value of the attribute. + type: string + required: + - key + - value + type: object + type: array + port: + description: The port of the listener + format: int32 + type: integer + protocol: + description: The protocol of the listener + type: string + type: object + type: array loadBalancerAttributes: description: LoadBalancerAttributes define the custom attributes to LoadBalancers for all Ingress that that belong to IngressClass with @@ -91,50 +132,63 @@ spec: - value type: object type: array + minimumLoadBalancerCapacity: + description: MinimumLoadBalancerCapacity define the capacity reservation + for LoadBalancers for all Ingress that belong to IngressClass with + this IngressClassParams. + properties: + capacityUnits: + description: The Capacity Units Value. + format: int32 + type: integer + required: + - capacityUnits + type: object namespaceSelector: - description: NamespaceSelector restrict the namespaces of Ingresses - that are allowed to specify the IngressClass with this IngressClassParams. + description: |- + NamespaceSelector restrict the namespaces of Ingresses that are allowed to specify the IngressClass with this IngressClassParams. * if absent or present but empty, it selects all namespaces. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic @@ -167,10 +221,11 @@ spec: items: type: string type: array - description: Tags specifies subnets in the load balancer's VPC - where each tag specified in the map key contains one of the - values in the corresponding value list. Exactly one of this - or `ids` must be specified. + description: |- + Tags specifies subnets in the load balancer's VPC where each + tag specified in the map key contains one of the values in the corresponding + value list. + Exactly one of this or `ids` must be specified. type: object type: object tags: @@ -200,8 +255,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.1 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: targetgroupbindings.elbv2.k8s.aws spec: group: elbv2.k8s.aws @@ -230,6 +284,11 @@ spec: name: ARN priority: 1 type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -239,20 +298,29 @@ spec: description: TargetGroupBinding is the Schema for the TargetGroupBinding API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: description: TargetGroupBindingSpec defines the desired state of TargetGroupBinding properties: + multiClusterTargetGroup: + description: MultiClusterTargetGroup Denotes if the TargetGroup is + shared among multiple clusters + type: boolean networking: description: networking provides the networking setup for ELBV2 LoadBalancer to access targets in TargetGroup. @@ -263,28 +331,30 @@ spec: items: properties: from: - description: List of peers which should be able to access - the targets in TargetGroup. At least one NetworkingPeer - should be specified. + description: |- + List of peers which should be able to access the targets in TargetGroup. + At least one NetworkingPeer should be specified. items: description: NetworkingPeer defines the source/destination peer for networking rules. properties: ipBlock: - description: IPBlock defines an IPBlock peer. If specified, - none of the other fields can be set. + description: |- + IPBlock defines an IPBlock peer. + If specified, none of the other fields can be set. properties: cidr: - description: CIDR is the network CIDR. Both IPV4 - or IPV6 CIDR are accepted. + description: |- + CIDR is the network CIDR. + Both IPV4 or IPV6 CIDR are accepted. type: string required: - cidr type: object securityGroup: - description: SecurityGroup defines a SecurityGroup - peer. If specified, none of the other fields can - be set. + description: |- + SecurityGroup defines a SecurityGroup peer. + If specified, none of the other fields can be set. properties: groupID: description: GroupID is the EC2 SecurityGroupID. @@ -295,24 +365,24 @@ spec: type: object type: array ports: - description: List of ports which should be made accessible - on the targets in TargetGroup. If ports is empty or unspecified, - it defaults to all ports with TCP. + description: |- + List of ports which should be made accessible on the targets in TargetGroup. + If ports is empty or unspecified, it defaults to all ports with TCP. items: properties: port: anyOf: - type: integer - type: string - description: The port which traffic must match. When - NodePort endpoints(instance TargetType) is used, - this must be a numerical port. When Port endpoints(ip - TargetType) is used, this can be either numerical - or named port on pods. if port is unspecified, it - defaults to all ports. + description: |- + The port which traffic must match. + When NodePort endpoints(instance TargetType) is used, this must be a numerical port. + When Port endpoints(ip TargetType) is used, this can be either numerical or named port on pods. + if port is unspecified, it defaults to all ports. x-kubernetes-int-or-string: true protocol: - description: The protocol which traffic must match. + description: |- + The protocol which traffic must match. If protocol is unspecified, it defaults to TCP. enum: - TCP @@ -347,6 +417,9 @@ spec: description: targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. + type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, it will be automatically inferred. @@ -356,7 +429,6 @@ spec: type: string required: - serviceRef - - targetGroupARN type: object status: description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding @@ -389,6 +461,11 @@ spec: name: ARN priority: 1 type: string + - description: The AWS TargetGroup's Name + jsonPath: .spec.targetGroupName + name: NAME + priority: 2 + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -398,14 +475,19 @@ spec: description: TargetGroupBinding is the Schema for the TargetGroupBinding API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -419,6 +501,10 @@ spec: - ipv4 - ipv6 type: string + multiClusterTargetGroup: + description: MultiClusterTargetGroup Denotes if the TargetGroup is + shared among multiple clusters + type: boolean networking: description: networking defines the networking rules to allow ELBV2 LoadBalancer to access targets in TargetGroup. @@ -431,28 +517,30 @@ spec: of traffic that is allowed to access TargetGroup's targets. properties: from: - description: List of peers which should be able to access - the targets in TargetGroup. At least one NetworkingPeer - should be specified. + description: |- + List of peers which should be able to access the targets in TargetGroup. + At least one NetworkingPeer should be specified. items: description: NetworkingPeer defines the source/destination peer for networking rules. properties: ipBlock: - description: IPBlock defines an IPBlock peer. If specified, - none of the other fields can be set. + description: |- + IPBlock defines an IPBlock peer. + If specified, none of the other fields can be set. properties: cidr: - description: CIDR is the network CIDR. Both IPV4 - or IPV6 CIDR are accepted. + description: |- + CIDR is the network CIDR. + Both IPV4 or IPV6 CIDR are accepted. type: string required: - cidr type: object securityGroup: - description: SecurityGroup defines a SecurityGroup - peer. If specified, none of the other fields can - be set. + description: |- + SecurityGroup defines a SecurityGroup peer. + If specified, none of the other fields can be set. properties: groupID: description: GroupID is the EC2 SecurityGroupID. @@ -463,9 +551,9 @@ spec: type: object type: array ports: - description: List of ports which should be made accessible - on the targets in TargetGroup. If ports is empty or unspecified, - it defaults to all ports with TCP. + description: |- + List of ports which should be made accessible on the targets in TargetGroup. + If ports is empty or unspecified, it defaults to all ports with TCP. items: description: NetworkingPort defines the port and protocol for networking rules. @@ -474,15 +562,15 @@ spec: anyOf: - type: integer - type: string - description: The port which traffic must match. When - NodePort endpoints(instance TargetType) is used, - this must be a numerical port. When Port endpoints(ip - TargetType) is used, this can be either numerical - or named port on pods. if port is unspecified, it - defaults to all ports. + description: |- + The port which traffic must match. + When NodePort endpoints(instance TargetType) is used, this must be a numerical port. + When Port endpoints(ip TargetType) is used, this can be either numerical or named port on pods. + if port is unspecified, it defaults to all ports. x-kubernetes-int-or-string: true protocol: - description: The protocol which traffic must match. + description: |- + The protocol which traffic must match. If protocol is unspecified, it defaults to TCP. enum: - TCP @@ -504,41 +592,42 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object x-kubernetes-map-type: atomic @@ -562,7 +651,9 @@ spec: targetGroupARN: description: targetGroupARN is the Amazon Resource Name (ARN) for the TargetGroup. - minLength: 1 + type: string + targetGroupName: + description: targetGroupName is the Name of the TargetGroup. type: string targetType: description: targetType is the TargetType of TargetGroup. If unspecified, @@ -571,9 +662,12 @@ spec: - instance - ip type: string + vpcID: + description: VpcID is the VPC of the TargetGroup. If unspecified, + it will be automatically inferred. + type: string required: - serviceRef - - targetGroupARN type: object status: description: TargetGroupBindingStatus defines the observed state of TargetGroupBinding diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/_helpers.tpl b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/_helpers.tpl index 660f6ee9d..d916b99c4 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/_helpers.tpl +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/_helpers.tpl @@ -45,12 +45,14 @@ This enables using a shorter name for the resources, for example aws-load-balanc Common labels */}} {{- define "aws-load-balancer-controller.labels" -}} +{{- if eq (default "helm" .Values.creator) "helm" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} helm.sh/chart: {{ include "aws-load-balancer-controller.chart" . }} +{{- end }} {{ include "aws-load-balancer-controller.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} {{- if .Values.additionalLabels }} {{ toYaml .Values.additionalLabels }} {{- end -}} diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/deployment.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/deployment.yaml index e2b5225ff..4506d489e 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/deployment.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/deployment.yaml @@ -11,6 +11,7 @@ metadata: {{- include "aws-load-balancer-controller.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} selector: matchLabels: {{- include "aws-load-balancer-controller.selectorLabels" . | nindent 6 }} @@ -37,6 +38,9 @@ spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.runtimeClassName }} + runtimeClassName: {{ .Values.runtimeClassName }} {{- end }} serviceAccountName: {{ include "aws-load-balancer-controller.serviceAccountName" . }} volumes: @@ -58,15 +62,17 @@ spec: containers: - name: {{ .Chart.Name }} args: - - --cluster-name={{ required "Chart cannot be installed without a valid clusterName!" .Values.clusterName }} + - --cluster-name={{ required "Chart cannot be installed without a valid clusterName!" (tpl (default "" .Values.clusterName) .) }} {{- if .Values.ingressClass }} - --ingress-class={{ .Values.ingressClass }} {{- end }} - {{- if .Values.region }} + {{- $region := tpl (default "" .Values.region) . }} + {{- if $region }} - --aws-region={{ .Values.region }} {{- end }} - {{- if .Values.vpcId }} - - --aws-vpc-id={{ .Values.vpcId }} + {{- $vpcID := tpl (default "" .Values.vpcId) . }} + {{- if $vpcID }} + - --aws-vpc-id={{ $vpcID }} {{- end }} {{- if .Values.awsApiEndpoints }} - --aws-api-endpoints={{ .Values.awsApiEndpoints }} @@ -104,6 +110,9 @@ spec: {{- if .Values.targetgroupbindingMaxExponentialBackoffDelay }} - --targetgroupbinding-max-exponential-backoff-delay={{ .Values.targetgroupbindingMaxExponentialBackoffDelay }} {{- end }} + {{- if .Values.lbStabilizationMonitorInterval }} + - --lb-stabilization-monitor-interval={{ .Values.lbStabilizationMonitorInterval }} + {{- end }} {{- if .Values.logLevel }} - --log-level={{ .Values.logLevel }} {{- end }} @@ -122,6 +131,12 @@ spec: {{- if kindIs "bool" .Values.disableIngressGroupNameAnnotation }} - --disable-ingress-group-name-annotation={{ .Values.disableIngressGroupNameAnnotation }} {{- end }} + {{- if kindIs "bool" .Values.tolerateNonExistentBackendService }} + - --tolerate-non-existent-backend-service={{ .Values.tolerateNonExistentBackendService }} + {{- end }} + {{- if kindIs "bool" .Values.tolerateNonExistentBackendAction }} + - --tolerate-non-existent-backend-action={{ .Values.tolerateNonExistentBackendAction }} + {{- end }} {{- if .Values.defaultSSLPolicy }} - --default-ssl-policy={{ .Values.defaultSSLPolicy }} {{- end }} @@ -149,13 +164,42 @@ spec: {{- if ne .Values.defaultTargetType "instance" }} - --default-target-type={{ .Values.defaultTargetType }} {{- end }} - {{- if .Values.env }} + {{- if .Values.serviceTargetENISGTags }} + - --service-target-eni-security-group-tags={{ .Values.serviceTargetENISGTags }} + {{- end }} + {{- if .Values.certDiscovery.allowedCertificateAuthorityARNs }} + - --allowed-certificate-authority-arns={{ .Values.certDiscovery.allowedCertificateAuthorityARNs }} + {{- end }} + {{- if .Values.loadBalancerClass }} + - --load-balancer-class={{ .Values.loadBalancerClass }} + {{- end }} + {{- if or .Values.env .Values.envSecretName }} env: + {{- if .Values.env}} {{- range $key, $value := .Values.env }} - name: {{ $key }} value: "{{ $value }}" {{- end }} {{- end }} + {{- if .Values.envSecretName }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.envSecretName }} + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.envSecretName }} + key: access_key + optional: true + {{- end }} + {{- end }} + {{- if .Values.envFrom }} + envFrom: + {{- toYaml .Values.envFrom | nindent 10 }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 10 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -180,6 +224,10 @@ spec: livenessProbe: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} {{- with .Values.nodeSelector }} nodeSelector: diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/hpa.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/hpa.yaml new file mode 100644 index 000000000..68689ba66 --- /dev/null +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/hpa.yaml @@ -0,0 +1,34 @@ +{{- if .Values.autoscaling.enabled }} +{{- if (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version)}} +apiVersion: autoscaling/v2 +{{- else }} +apiVersion: autoscaling/v2beta2 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "aws-load-balancer-controller.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-load-balancer-controller.labels" . | nindent 4 }} + annotations: + {{- .Values.annotations | toYaml | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "aws-load-balancer-controller.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ required "A valid .Values.autoscaling.maxReplicas value is required" .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + type: Utilization + {{- end }} + {{- if .Values.autoscaling.autoscaleBehavior }} + behavior: {{ toYaml .Values.autoscaling.autoscaleBehavior | nindent 4 }} + {{- end }} +{{- end }} diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/rbac.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/rbac.yaml index fc3bda695..0dcc68c77 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/rbac.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/rbac.yaml @@ -75,6 +75,9 @@ rules: - apiGroups: [""] resources: [nodes, namespaces, endpoints] verbs: [get, list, watch] +- apiGroups: [""] + resources: [configmaps] + verbs: [get, delete, create, update] {{- if .Values.clusterSecretsPermissions.allowAllSecrets }} - apiGroups: [""] resources: [secrets] diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/servicemonitor.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/servicemonitor.yaml index c811be253..0454558c2 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/servicemonitor.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/servicemonitor.yaml @@ -3,18 +3,14 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: {{ include "aws-load-balancer-controller.fullname" . }} - {{- if .Values.serviceMonitor.namespace }} - namespace: {{ .Values.serviceMonitor.namespace }} - {{- else }} - namespace: {{ .Release.Namespace | quote }} - {{- end }} + namespace: {{ default .Release.Namespace .Values.serviceMonitor.namespace }} labels: {{- include "aws-load-balancer-controller.labels" . | nindent 4 }} - {{- with .Values.serviceMonitor.additionalLabels }} + {{- with .Values.serviceMonitor.additionalLabels }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: - jobLabel: {{ .Release.Name }} + jobLabel: app.kubernetes.io/instance namespaceSelector: matchNames: - {{ .Release.Namespace }} @@ -29,7 +25,19 @@ spec: endpoints: - port: metrics-server path: /metrics - {{- with .Values.serviceMonitor.interval }} + scheme: http + {{- with .Values.serviceMonitor.interval }} interval: {{ . }} - {{- end }} -{{- end -}} \ No newline at end of file + {{- end }} + {{- with .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/webhook.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/webhook.yaml index e7d557e41..504f08ccb 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/templates/webhook.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/templates/webhook.yaml @@ -65,7 +65,7 @@ webhooks: name: {{ template "aws-load-balancer-controller.webhookService" . }} namespace: {{ $.Release.Namespace }} path: /mutate-v1-service - failurePolicy: Fail + failurePolicy: {{ .Values.serviceMutatorWebhookConfig.failurePolicy }} name: mservice.elbv2.k8s.aws admissionReviewVersions: - v1beta1 @@ -75,13 +75,21 @@ webhooks: operator: NotIn values: - {{ include "aws-load-balancer-controller.name" . }} + {{- if .Values.serviceMutatorWebhookConfig.objectSelector.matchExpressions }} + {{- toYaml .Values.serviceMutatorWebhookConfig.objectSelector.matchExpressions | nindent 4 }} + {{- end }} + + {{- if .Values.serviceMutatorWebhookConfig.objectSelector.matchLabels }} + matchLabels: + {{- toYaml .Values.serviceMutatorWebhookConfig.objectSelector.matchLabels | nindent 6 }} + {{- end }} rules: - apiGroups: - "" apiVersions: - v1 operations: - - CREATE + {{- toYaml .Values.serviceMutatorWebhookConfig.operations | nindent 4 }} resources: - services sideEffects: None @@ -173,6 +181,7 @@ webhooks: resources: - targetgroupbindings sideEffects: None +{{- if not $.Values.webhookConfig.disableIngressValidation }} - clientConfig: {{ if not $.Values.enableCertManager -}} caBundle: {{ $tls.caCert }} @@ -197,6 +206,7 @@ webhooks: resources: - ingresses sideEffects: None +{{- end }} --- {{- if not $.Values.enableCertManager }} apiVersion: v1 diff --git a/internal/constellation/helm/charts/aws-load-balancer-controller/values.yaml b/internal/constellation/helm/charts/aws-load-balancer-controller/values.yaml index dea199559..c2f465bcd 100644 --- a/internal/constellation/helm/charts/aws-load-balancer-controller/values.yaml +++ b/internal/constellation/helm/charts/aws-load-balancer-controller/values.yaml @@ -4,15 +4,29 @@ replicaCount: 2 +revisionHistoryLimit: 10 + image: repository: public.ecr.aws/eks/aws-load-balancer-controller - tag: v2.5.3 + tag: v2.11.0 pullPolicy: IfNotPresent +runtimeClassName: "" imagePullSecrets: [] nameOverride: "" fullnameOverride: "" +# AWS LBC only has 1 main working pod, other pods are just standby +# the purpose of enable hpa is to survive load induced failure by the calls to the aws-load-balancer-webhook-service +# since the calls from kube-apiserver are sent round-robin to all replicas, and the failure policy on those webhooks is Fail +# if the pods become overloaded and do not respond within the timeout that could block the creation of pods, targetgroupbindings or ingresses +# Please keep in mind that the controller pods have `priorityClassName: system-cluster-critical`, enabling HPA may lead to the eviction of other low-priority pods in the node +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + serviceAccount: # Specifies whether a service account should be created create: true @@ -106,8 +120,8 @@ clusterName: # cluster contains configurations specific to the kubernetes cluster cluster: - # Cluster DNS domain (required for requesting TLS certificates) - dnsDomain: cluster.local + # Cluster DNS domain (required for requesting TLS certificates) + dnsDomain: cluster.local # The ingress class this controller will satisfy. If not specified, controller will match all # ingresses without ingress class annotation and ingresses of type alb @@ -187,6 +201,10 @@ logLevel: # The address the metric endpoint binds to. (default ":8080") metricsBindAddr: "" +webhookConfig: + # disableIngressValidation disables the validation of resources of kind Ingress, false by default + disableIngressValidation: + # The TCP port the Webhook server binds to. (default 9443) webhookBindPort: @@ -196,7 +214,7 @@ webhookTLS: cert: key: -# array of namespace selectors for the webhook +# array of namespace selectors for the pod mutator webhook webhookNamespaceSelectors: # - key: elbv2.k8s.aws/pod-readiness-gate-inject # operator: In @@ -215,7 +233,10 @@ targetgroupbindingMaxConcurrentReconciles: # Maximum duration of exponential backoff for targetGroupBinding reconcile failures targetgroupbindingMaxExponentialBackoffDelay: -# Period at which the controller forces the repopulation of its local object stores. (default 1h0m0s) +# Interval at which the controller monitors the state of load balancer after creation for stabilization +lbStabilizationMonitorInterval: + +# Period at which the controller forces the repopulation of its local object stores. (default 10h0m0s) syncPeriod: # Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. @@ -227,6 +248,12 @@ disableIngressClassAnnotation: # disableIngressGroupNameAnnotation disables the usage of alb.ingress.kubernetes.io/group.name annotation, false by default disableIngressGroupNameAnnotation: +# tolerateNonExistentBackendService permits rules which specify backend services that don't exist, true by default (When enabled, it will return 503 error if backend service not exist) +tolerateNonExistentBackendService: + +# tolerateNonExistentBackendAction permits rules which specify backend actions that don't exist, true by default (When enabled, it will return 503 error if backend action not exist) +tolerateNonExistentBackendAction: + # defaultSSLPolicy specifies the default SSL policy to use for TLS/HTTPS listeners defaultSSLPolicy: @@ -240,6 +267,17 @@ livenessProbe: initialDelaySeconds: 30 timeoutSeconds: 10 +# readiness probe configuration for the controller +readinessProbe: + failureThreshold: 2 + httpGet: + path: /readyz + port: 61779 + scheme: HTTP + successThreshold: 1 + initialDelaySeconds: 10 + timeoutSeconds: 10 + # Environment variables to set for aws-load-balancer-controller pod. # We strongly discourage programming access credentials in the controller environment. You should setup IRSA or # comparable solutions like kube2iam, kiam etc instead. @@ -247,8 +285,15 @@ env: # ENV_1: "" # ENV_2: "" +# Use Environment variables credentials from Secret (aws-secret) for aws-load-balancer-controller pod similarly as The EBS CSI Driver does. +# envSecretName: aws-secret + +# Use envFrom to set environment variables from a Secret or ConfigMap +# envFrom: +# - secretRef: +# name: my-secret + # Specifies if aws-load-balancer-controller should be started in hostNetwork mode. -# # This is required if using a custom CNI where the managed control plane nodes are unable to initiate # network connections to the pods, for example using Calico CNI plugin on EKS. This is not required or # recommended if using the Amazon VPC CNI plugin. @@ -315,6 +360,11 @@ controllerConfig: # EnableIPTargetType: true # SubnetsClusterTagCheck: true # NLBHealthCheckAdvancedConfig: true + # ALBSingleSubnet: false + # LBCapacityReservation: true + +certDiscovery: + allowedCertificateAuthorityARNs: "" # empty means all CAs are in scope # objectSelector for webhook objectSelector: @@ -329,12 +379,18 @@ objectSelector: serviceMonitor: # Specifies whether a service monitor should be created enabled: false - # Labels to add to the service account + # Namespace to create the service monitor in + namespace: + # Labels to add to the service monitor additionalLabels: {} # Prometheus scrape interval interval: 1m - # Namespace to create the service monitor in - namespace: + # Prometheus scrape timeout + scrapeTimeout: + # Relabelings to apply to samples before ingestion + relabelings: + # Metric relabelings to apply to samples before ingestion + metricRelabelings: # clusterSecretsPermissions lets you configure RBAC permissions for secret resources # Access to secrets resource is required only if you use the OIDC feature, and instead of @@ -351,3 +407,30 @@ ingressClassConfig: # enableServiceMutatorWebhook allows you enable the webhook which makes this controller the default for all new services of type LoadBalancer enableServiceMutatorWebhook: true + +# serviceMutatorWebhook contains configurations specific to the service mutator webhook +serviceMutatorWebhookConfig: + # whether or not to fail the service creation if the webhook fails + failurePolicy: Fail + # limit webhook to only mutate services matching the objectSelector + objectSelector: + matchExpressions: [] + # - key: + # operator: + # values: + # - + matchLabels: {} + # key: value + # which operations trigger the webhook + operations: + - CREATE + # - UPDATE + +# serviceTargetENISGTags specifies AWS tags, in addition to the cluster tags, for finding the target ENI SG to which to add inbound rules from NLBs. +serviceTargetENISGTags: + +# Specifies the class of load balancer to use for services. This affects how services are provisioned if type LoadBalancer is used (default service.k8s.aws/nlb) +loadBalancerClass: + +# creator will disable helm default labels, so you can only add yours +# creator: "me" diff --git a/internal/constellation/helm/charts/cert-manager/Chart.yaml b/internal/constellation/helm/charts/cert-manager/Chart.yaml index 7a8e8043b..aea96934f 100644 --- a/internal/constellation/helm/charts/cert-manager/Chart.yaml +++ b/internal/constellation/helm/charts/cert-manager/Chart.yaml @@ -1,13 +1,15 @@ annotations: + artifacthub.io/category: security + artifacthub.io/license: Apache-2.0 artifacthub.io/prerelease: "false" artifacthub.io/signKey: | fingerprint: 1020CF3C033D4F35BAE1C19E1226061C665DF13E url: https://cert-manager.io/public-keys/cert-manager-keyring-2021-09-20-1020CF3C033D4F35BAE1C19E1226061C665DF13E.gpg -apiVersion: v1 -appVersion: v1.12.6 +apiVersion: v2 +appVersion: v1.15.0 description: A Helm chart for cert-manager -home: https://github.com/cert-manager/cert-manager -icon: https://raw.githubusercontent.com/cert-manager/cert-manager/d53c0b9270f8cd90d908460d69502694e1838f5f/logo/logo-small.png +home: https://cert-manager.io +icon: https://raw.githubusercontent.com/cert-manager/community/4d35a69437d21b76322157e6284be4cd64e6d2b7/logo/logo-small.png keywords: - cert-manager - kube-lego @@ -21,4 +23,4 @@ maintainers: name: cert-manager sources: - https://github.com/cert-manager/cert-manager -version: v1.12.6 +version: v1.15.0 diff --git a/internal/constellation/helm/charts/cert-manager/templates/NOTES.txt b/internal/constellation/helm/charts/cert-manager/templates/NOTES.txt index 102535460..341d10123 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/NOTES.txt +++ b/internal/constellation/helm/charts/cert-manager/templates/NOTES.txt @@ -1,3 +1,6 @@ +{{- if .Values.installCRDs }} +⚠️ WARNING: `installCRDs` is deprecated, use `crds.enabled` instead. +{{- end }} cert-manager {{ .Chart.AppVersion }} has been deployed successfully! In order to begin issuing certificates, you will need to set up a ClusterIssuer diff --git a/internal/constellation/helm/charts/cert-manager/templates/_helpers.tpl b/internal/constellation/helm/charts/cert-manager/templates/_helpers.tpl index 90db4af26..9902c089f 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/_helpers.tpl +++ b/internal/constellation/helm/charts/cert-manager/templates/_helpers.tpl @@ -172,3 +172,31 @@ https://github.com/helm/helm/issues/5358 {{- define "cert-manager.namespace" -}} {{ .Values.namespace | default .Release.Namespace }} {{- end -}} + +{{/* +Util function for generating the image URL based on the provided options. +IMPORTANT: This function is standarized across all charts in the cert-manager GH organization. +Any changes to this function should also be made in cert-manager, trust-manager, approver-policy, ... +See https://github.com/cert-manager/cert-manager/issues/6329 for a list of linked PRs. +*/}} +{{- define "image" -}} +{{- $defaultTag := index . 1 -}} +{{- with index . 0 -}} +{{- if .registry -}}{{ printf "%s/%s" .registry .repository }}{{- else -}}{{- .repository -}}{{- end -}} +{{- if .digest -}}{{ printf "@%s" .digest }}{{- else -}}{{ printf ":%s" (default $defaultTag .tag) }}{{- end -}} +{{- end }} +{{- end }} + +{{/* +Check that the user has not set both .installCRDs and .crds.enabled or +set .installCRDs and disabled .crds.keep. +.installCRDs is deprecated and users should use .crds.enabled and .crds.keep instead. +*/}} +{{- define "cert-manager.crd-check" -}} + {{- if and (.Values.installCRDs) (.Values.crds.enabled) }} + {{- fail "ERROR: the deprecated .installCRDs option cannot be enabled at the same time as its replacement .crds.enabled" }} + {{- end }} + {{- if and (.Values.installCRDs) (not .Values.crds.keep) }} + {{- fail "ERROR: .crds.keep is not compatible with .installCRDs, please use .crds.enabled and .crds.keep instead" }} + {{- end }} +{{- end -}} diff --git a/internal/constellation/helm/charts/cert-manager/templates/cainjector-config.yaml b/internal/constellation/helm/charts/cert-manager/templates/cainjector-config.yaml new file mode 100644 index 000000000..82399cc1a --- /dev/null +++ b/internal/constellation/helm/charts/cert-manager/templates/cainjector-config.yaml @@ -0,0 +1,18 @@ +{{- if .Values.cainjector.config -}} +{{- $_ := .Values.cainjector.config.apiVersion | required ".Values.cainjector.config.apiVersion must be set !" -}} +{{- $_ := .Values.cainjector.config.kind | required ".Values.cainjector.config.kind must be set !" -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cainjector.fullname" . }} + namespace: {{ include "cert-manager.namespace" . }} + labels: + app: {{ include "cainjector.name" . }} + app.kubernetes.io/name: {{ include "cainjector.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: "cainjector" + {{- include "labels" . | nindent 4 }} +data: + config.yaml: | + {{- .Values.cainjector.config | toYaml | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/internal/constellation/helm/charts/cert-manager/templates/cainjector-deployment.yaml b/internal/constellation/helm/charts/cert-manager/templates/cainjector-deployment.yaml index 122017374..8f9f7f331 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/cainjector-deployment.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/cainjector-deployment.yaml @@ -16,6 +16,10 @@ metadata: {{- end }} spec: replicas: {{ .Values.cainjector.replicaCount }} + {{- /* The if statement below is equivalent to {{- if $value }} but will also return true for 0. */ -}} + {{- if not (has (quote .Values.global.revisionHistoryLimit) (list "" (quote ""))) }} + revisionHistoryLimit: {{ .Values.global.revisionHistoryLimit }} + {{- end }} selector: matchLabels: app.kubernetes.io/name: {{ include "cainjector.name" . }} @@ -45,6 +49,7 @@ spec: {{- if hasKey .Values.cainjector "automountServiceAccountToken" }} automountServiceAccountToken: {{ .Values.cainjector.automountServiceAccountToken }} {{- end }} + enableServiceLinks: {{ .Values.cainjector.enableServiceLinks }} {{- with .Values.global.priorityClassName }} priorityClassName: {{ . | quote }} {{- end }} @@ -54,14 +59,16 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-cainjector - {{- with .Values.cainjector.image }} - image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}" - {{- end }} + image: "{{ template "image" (tuple .Values.cainjector.image $.Chart.AppVersion) }}" imagePullPolicy: {{ .Values.cainjector.image.pullPolicy }} args: - {{- if .Values.global.logLevel }} + {{- /* The if statement below is equivalent to {{- if $value }} but will also return true for 0. */ -}} + {{- if not (has (quote .Values.global.logLevel) (list "" (quote ""))) }} - --v={{ .Values.global.logLevel }} {{- end }} + {{- if .Values.cainjector.config }} + - --config=/var/cert-manager/config/config.yaml + {{- end }} {{- with .Values.global.leaderElection }} - --leader-election-namespace={{ .namespace }} {{- if .leaseDuration }} @@ -74,6 +81,9 @@ spec: - --leader-election-retry-period={{ .retryPeriod }} {{- end }} {{- end }} + {{- with .Values.cainjector.featureGates}} + - --feature-gates={{ . }} + {{- end}} {{- with .Values.cainjector.extraArgs }} {{- toYaml . | nindent 10 }} {{- end }} @@ -90,9 +100,15 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.cainjector.volumeMounts }} + {{- if or .Values.cainjector.config .Values.cainjector.volumeMounts }} volumeMounts: + {{- if .Values.cainjector.config }} + - name: config + mountPath: /var/cert-manager/config + {{- end }} + {{- with .Values.cainjector.volumeMounts }} {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} {{- with .Values.cainjector.nodeSelector }} nodeSelector: @@ -110,8 +126,15 @@ spec: topologySpreadConstraints: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.cainjector.volumes }} + {{- if or .Values.cainjector.volumes .Values.cainjector.config }} volumes: + {{- if .Values.cainjector.config }} + - name: config + configMap: + name: {{ include "cainjector.fullname" . }} + {{- end }} + {{ with .Values.cainjector.volumes }} {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/cainjector-poddisruptionbudget.yaml b/internal/constellation/helm/charts/cert-manager/templates/cainjector-poddisruptionbudget.yaml index f080b753a..6a7d60913 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/cainjector-poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/cainjector-poddisruptionbudget.yaml @@ -17,10 +17,13 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: "cainjector" - {{- with .Values.cainjector.podDisruptionBudget.minAvailable }} - minAvailable: {{ . }} + {{- if not (or (hasKey .Values.cainjector.podDisruptionBudget "minAvailable") (hasKey .Values.cainjector.podDisruptionBudget "maxUnavailable")) }} + minAvailable: 1 # Default value because minAvailable and maxUnavailable are not set {{- end }} - {{- with .Values.cainjector.podDisruptionBudget.maxUnavailable }} - maxUnavailable: {{ . }} + {{- if hasKey .Values.cainjector.podDisruptionBudget "minAvailable" }} + minAvailable: {{ .Values.cainjector.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if hasKey .Values.cainjector.podDisruptionBudget "maxUnavailable" }} + maxUnavailable: {{ .Values.cainjector.podDisruptionBudget.maxUnavailable }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/controller-config.yaml b/internal/constellation/helm/charts/cert-manager/templates/controller-config.yaml index a1b337572..25f62ef1d 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/controller-config.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/controller-config.yaml @@ -1,12 +1,6 @@ {{- if .Values.config -}} - {{- if not .Values.config.apiVersion -}} - {{- fail "config.apiVersion must be set" -}} - {{- end -}} - - {{- if not .Values.config.kind -}} - {{- fail "config.kind must be set" -}} - {{- end -}} -{{- end -}} +{{- $_ := .Values.config.apiVersion | required ".Values.config.apiVersion must be set !" -}} +{{- $_ := .Values.config.kind | required ".Values.config.kind must be set !" -}} apiVersion: v1 kind: ConfigMap metadata: @@ -19,7 +13,6 @@ metadata: app.kubernetes.io/component: "controller" {{- include "labels" . | nindent 4 }} data: - {{- if .Values.config }} config.yaml: | - {{ .Values.config | toYaml | nindent 4 }} - {{- end }} + {{- .Values.config | toYaml | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/internal/constellation/helm/charts/cert-manager/templates/crds.yaml b/internal/constellation/helm/charts/cert-manager/templates/crds.yaml index 820698742..2c70ca34c 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/crds.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/crds.yaml @@ -1,8 +1,13 @@ -{{- if .Values.installCRDs }} +# {{- include "cert-manager.crd-check" . }} +# START crd {{- if or .Values.crds.enabled .Values.installCRDs }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: certificaterequests.cert-manager.io + # START annotations {{- if .Values.crds.keep }} + annotations: + helm.sh/resource-policy: keep + # END annotations {{- end }} labels: app: '{{ template "cert-manager.name" . }}' app.kubernetes.io/name: '{{ template "cert-manager.name" . }}' @@ -51,47 +56,91 @@ spec: type: date schema: openAPIV3Schema: - description: "A CertificateRequest is used to request a signed certificate from one of the configured issuers. \n All fields within the CertificateRequest's `spec` are immutable after creation. A CertificateRequest will either succeed or fail, as denoted by its `status.state` field. \n A CertificateRequest is a one-shot resource, meaning it represents a single point in time request for a certificate and cannot be re-used." + description: |- + A CertificateRequest is used to request a signed certificate from one of the + configured issuers. + + + All fields within the CertificateRequest's `spec` are immutable after creation. + A CertificateRequest will either succeed or fail, as denoted by its `Ready` status + condition and its `status.failureTime` field. + + + A CertificateRequest is a one-shot resource, meaning it represents a single + point in time request for a certificate and cannot be re-used. type: object - required: - - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: Desired state of the CertificateRequest resource. + description: |- + Specification of the desired state of the CertificateRequest resource. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object required: - issuerRef - request properties: duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. This option may be ignored/overridden by some issuer types. + description: |- + Requested 'duration' (i.e. lifetime) of the Certificate. Note that the + issuer may choose to ignore the requested duration, just like any other + requested attribute. type: string extra: - description: Extra contains extra attributes of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + description: |- + Extra contains extra attributes of the user that created the CertificateRequest. + Populated by the cert-manager webhook on creation and immutable. type: object additionalProperties: type: array items: type: string groups: - description: Groups contains group membership of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + description: |- + Groups contains group membership of the user that created the CertificateRequest. + Populated by the cert-manager webhook on creation and immutable. type: array items: type: string x-kubernetes-list-type: atomic isCA: - description: IsCA will request to mark the certificate as valid for certificate signing when submitting to the issuer. This will automatically add the `cert sign` usage to the list of `usages`. + description: |- + Requested basic constraints isCA value. Note that the issuer may choose + to ignore the requested isCA value, just like any other requested attribute. + + + NOTE: If the CSR in the `Request` field has a BasicConstraints extension, + it must have the same isCA value as specified here. + + + If true, this will automatically add the `cert sign` usage to the list + of requested `usages`. type: boolean issuerRef: - description: IssuerRef is a reference to the issuer for this CertificateRequest. If the `kind` field is not set, or set to `Issuer`, an Issuer resource with the given name in the same namespace as the CertificateRequest will be used. If the `kind` field is set to `ClusterIssuer`, a ClusterIssuer with the provided name will be used. The `name` field in this stanza is required at all times. The group field refers to the API group of the issuer which defaults to `cert-manager.io` if empty. + description: |- + Reference to the issuer responsible for issuing the certificate. + If the issuer is namespace-scoped, it must be in the same namespace + as the Certificate. If the issuer is cluster-scoped, it can be used + from any namespace. + + + The `name` field of the reference must always be specified. type: object required: - name @@ -106,17 +155,69 @@ spec: description: Name of the resource being referred to. type: string request: - description: The PEM-encoded x509 certificate signing request to be submitted to the CA for signing. + description: |- + The PEM-encoded X.509 certificate signing request to be submitted to the + issuer for signing. + + + If the CSR has a BasicConstraints extension, its isCA attribute must + match the `isCA` value of this CertificateRequest. + If the CSR has a KeyUsage extension, its key usages must match the + key usages in the `usages` field of this CertificateRequest. + If the CSR has a ExtKeyUsage extension, its extended key usages + must match the extended key usages in the `usages` field of this + CertificateRequest. type: string format: byte uid: - description: UID contains the uid of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + description: |- + UID contains the uid of the user that created the CertificateRequest. + Populated by the cert-manager webhook on creation and immutable. type: string usages: - description: Usages is the set of x509 usages that are requested for the certificate. If usages are set they SHOULD be encoded inside the CSR spec Defaults to `digital signature` and `key encipherment` if not specified. + description: |- + Requested key usages and extended key usages. + + + NOTE: If the CSR in the `Request` field has uses the KeyUsage or + ExtKeyUsage extension, these extensions must have the same values + as specified here without any additional values. + + + If unset, defaults to `digital signature` and `key encipherment`. type: array items: - description: "KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 \n Valid KeyUsage values are as follows: \"signing\", \"digital signature\", \"content commitment\", \"key encipherment\", \"key agreement\", \"data encipherment\", \"cert sign\", \"crl sign\", \"encipher only\", \"decipher only\", \"any\", \"server auth\", \"client auth\", \"code signing\", \"email protection\", \"s/mime\", \"ipsec end system\", \"ipsec tunnel\", \"ipsec user\", \"timestamping\", \"ocsp signing\", \"microsoft sgc\", \"netscape sgc\"" + description: |- + KeyUsage specifies valid usage contexts for keys. + See: + https://tools.ietf.org/html/rfc5280#section-4.2.1.3 + https://tools.ietf.org/html/rfc5280#section-4.2.1.12 + + + Valid KeyUsage values are as follows: + "signing", + "digital signature", + "content commitment", + "key encipherment", + "key agreement", + "data encipherment", + "cert sign", + "crl sign", + "encipher only", + "decipher only", + "any", + "server auth", + "client auth", + "code signing", + "email protection", + "s/mime", + "ipsec end system", + "ipsec tunnel", + "ipsec user", + "timestamping", + "ocsp signing", + "microsoft sgc", + "netscape sgc" type: string enum: - signing @@ -143,22 +244,39 @@ spec: - microsoft sgc - netscape sgc username: - description: Username contains the name of the user that created the CertificateRequest. Populated by the cert-manager webhook on creation and immutable. + description: |- + Username contains the name of the user that created the CertificateRequest. + Populated by the cert-manager webhook on creation and immutable. type: string status: - description: Status of the CertificateRequest. This is set and managed automatically. + description: |- + Status of the CertificateRequest. + This is set and managed automatically. + Read-only. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object properties: ca: - description: The PEM encoded x509 certificate of the signer, also known as the CA (Certificate Authority). This is set on a best-effort basis by different issuers. If not set, the CA is assumed to be unknown/not available. + description: |- + The PEM encoded X.509 certificate of the signer, also known as the CA + (Certificate Authority). + This is set on a best-effort basis by different issuers. + If not set, the CA is assumed to be unknown/not available. type: string format: byte certificate: - description: The PEM encoded x509 certificate resulting from the certificate signing request. If not set, the CertificateRequest has either not been completed or has failed. More information on failure can be found by checking the `conditions` field. + description: |- + The PEM encoded X.509 certificate resulting from the certificate + signing request. + If not set, the CertificateRequest has either not been completed or has + failed. More information on failure can be found by checking the + `conditions` field. type: string format: byte conditions: - description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready` and `InvalidRequest`. + description: |- + List of status conditions to indicate the status of a CertificateRequest. + Known condition types are `Ready`, `InvalidRequest`, `Approved` and `Denied`. type: array items: description: CertificateRequestCondition contains condition information for a CertificateRequest. @@ -168,14 +286,20 @@ spec: - type properties: lastTransitionTime: - description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + description: |- + LastTransitionTime is the timestamp corresponding to the last status + change of this condition. type: string format: date-time message: - description: Message is a human readable description of the details of the last transition, complementing reason. + description: |- + Message is a human readable description of the details of the last + transition, complementing reason. type: string reason: - description: Reason is a brief machine readable explanation for the condition's last transition. + description: |- + Reason is a brief machine readable explanation for the condition's last + transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). @@ -185,22 +309,34 @@ spec: - "False" - Unknown type: - description: Type of the condition, known values are (`Ready`, `InvalidRequest`, `Approved`, `Denied`). + description: |- + Type of the condition, known values are (`Ready`, `InvalidRequest`, + `Approved`, `Denied`). type: string x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map failureTime: - description: FailureTime stores the time that this CertificateRequest failed. This is used to influence garbage collection and back-off. + description: |- + FailureTime stores the time that this CertificateRequest failed. This is + used to influence garbage collection and back-off. type: string format: date-time served: true storage: true + +# END crd {{- end }} + --- +# START crd {{- if or .Values.crds.enabled .Values.installCRDs }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: certificates.cert-manager.io + # START annotations {{- if .Values.crds.keep }} + annotations: + helm.sh/resource-policy: keep + # END annotations {{- end }} labels: app: '{{ template "cert-manager.name" . }}' app.kubernetes.io/name: '{{ template "cert-manager.name" . }}' @@ -244,70 +380,132 @@ spec: type: date schema: openAPIV3Schema: - description: "A Certificate resource should be created to ensure an up to date and signed x509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. \n The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`)." + description: |- + A Certificate resource should be created to ensure an up to date and signed + X.509 certificate is stored in the Kubernetes Secret resource named in `spec.secretName`. + + + The stored certificate will be renewed before it expires (as configured by `spec.renewBefore`). type: object - required: - - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: Desired state of the Certificate resource. + description: |- + Specification of the desired state of the Certificate resource. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object required: - issuerRef - secretName properties: additionalOutputFormats: - description: AdditionalOutputFormats defines extra output formats of the private key and signed certificate chain to be written to this Certificate's target Secret. This is an Alpha Feature and is only enabled with the `--feature-gates=AdditionalCertificateOutputFormats=true` option on both the controller and webhook components. + description: |- + Defines extra output formats of the private key and signed certificate chain + to be written to this Certificate's target Secret. + + + This is a Beta Feature enabled by default. It can be disabled with the + `--feature-gates=AdditionalCertificateOutputFormats=false` option set on both + the controller and webhook components. type: array items: - description: CertificateAdditionalOutputFormat defines an additional output format of a Certificate resource. These contain supplementary data formats of the signed certificate chain and paired private key. + description: |- + CertificateAdditionalOutputFormat defines an additional output format of a + Certificate resource. These contain supplementary data formats of the signed + certificate chain and paired private key. type: object required: - type properties: type: - description: Type is the name of the format type that should be written to the Certificate's target Secret. + description: |- + Type is the name of the format type that should be written to the + Certificate's target Secret. type: string enum: - DER - CombinedPEM commonName: - description: 'CommonName is a common name to be used on the Certificate. The CommonName should have a length of 64 characters or fewer to avoid generating invalid CSRs. This value is ignored by TLS clients when any subject alt name is set. This is x509 behaviour: https://tools.ietf.org/html/rfc6125#section-6.4.4' + description: |- + Requested common name X509 certificate subject attribute. + More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 + NOTE: TLS clients will ignore this value when any subject alternative name is + set (see https://tools.ietf.org/html/rfc6125#section-6.4.4). + + + Should have a length of 64 characters or fewer to avoid generating invalid CSRs. + Cannot be set if the `literalSubject` field is set. type: string dnsNames: - description: DNSNames is a list of DNS subjectAltNames to be set on the Certificate. + description: Requested DNS subject alternative names. type: array items: type: string duration: - description: The requested 'duration' (i.e. lifetime) of the Certificate. This option may be ignored/overridden by some issuer types. If unset this defaults to 90 days. Certificate will be renewed either 2/3 through its duration or `renewBefore` period before its expiry, whichever is later. Minimum accepted duration is 1 hour. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration + description: |- + Requested 'duration' (i.e. lifetime) of the Certificate. Note that the + issuer may choose to ignore the requested duration, just like any other + requested attribute. + + + If unset, this defaults to 90 days. + Minimum accepted duration is 1 hour. + Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration. type: string emailAddresses: - description: EmailAddresses is a list of email subjectAltNames to be set on the Certificate. + description: Requested email subject alternative names. type: array items: type: string encodeUsagesInRequest: - description: EncodeUsagesInRequest controls whether key usages should be present in the CertificateRequest + description: |- + Whether the KeyUsage and ExtKeyUsage extensions should be set in the encoded CSR. + + + This option defaults to true, and should only be disabled if the target + issuer does not support CSRs with these X509 KeyUsage/ ExtKeyUsage extensions. type: boolean ipAddresses: - description: IPAddresses is a list of IP address subjectAltNames to be set on the Certificate. + description: Requested IP address subject alternative names. type: array items: type: string isCA: - description: IsCA will mark this Certificate as valid for certificate signing. This will automatically add the `cert sign` usage to the list of `usages`. + description: |- + Requested basic constraints isCA value. + The isCA value is used to set the `isCA` field on the created CertificateRequest + resources. Note that the issuer may choose to ignore the requested isCA value, just + like any other requested attribute. + + + If true, this will automatically add the `cert sign` usage to the list + of requested `usages`. type: boolean issuerRef: - description: IssuerRef is a reference to the issuer for this certificate. If the `kind` field is not set, or set to `Issuer`, an Issuer resource with the given name in the same namespace as the Certificate will be used. If the `kind` field is set to `ClusterIssuer`, a ClusterIssuer with the provided name will be used. The `name` field in this stanza is required at all times. + description: |- + Reference to the issuer responsible for issuing the certificate. + If the issuer is namespace-scoped, it must be in the same namespace + as the Certificate. If the issuer is cluster-scoped, it can be used + from any namespace. + + + The `name` field of the reference must always be specified. type: object required: - name @@ -322,94 +520,325 @@ spec: description: Name of the resource being referred to. type: string keystores: - description: Keystores configures additional keystore output formats stored in the `secretName` Secret resource. + description: Additional keystore output formats to be stored in the Certificate's Secret. type: object properties: jks: - description: JKS configures options for storing a JKS keystore in the `spec.secretName` Secret resource. + description: |- + JKS configures options for storing a JKS keystore in the + `spec.secretName` Secret resource. type: object required: - create - passwordSecretRef properties: + alias: + description: |- + Alias specifies the alias of the key in the keystore, required by the JKS format. + If not provided, the default alias `certificate` will be used. + type: string create: - description: Create enables JKS keystore creation for the Certificate. If true, a file named `keystore.jks` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.jks` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority + description: |- + Create enables JKS keystore creation for the Certificate. + If true, a file named `keystore.jks` will be created in the target + Secret resource, encrypted using the password stored in + `passwordSecretRef`. + The keystore file will be updated immediately. + If the issuer provided a CA certificate, a file named `truststore.jks` + will also be created in the target Secret resource, encrypted using the + password stored in `passwordSecretRef` + containing the issuing Certificate Authority type: boolean passwordSecretRef: - description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the JKS keystore. + description: |- + PasswordSecretRef is a reference to a key in a Secret resource + containing the password used to encrypt the JKS keystore. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string pkcs12: - description: PKCS12 configures options for storing a PKCS12 keystore in the `spec.secretName` Secret resource. + description: |- + PKCS12 configures options for storing a PKCS12 keystore in the + `spec.secretName` Secret resource. type: object required: - create - passwordSecretRef properties: create: - description: Create enables PKCS12 keystore creation for the Certificate. If true, a file named `keystore.p12` will be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef`. The keystore file will be updated immediately. If the issuer provided a CA certificate, a file named `truststore.p12` will also be created in the target Secret resource, encrypted using the password stored in `passwordSecretRef` containing the issuing Certificate Authority + description: |- + Create enables PKCS12 keystore creation for the Certificate. + If true, a file named `keystore.p12` will be created in the target + Secret resource, encrypted using the password stored in + `passwordSecretRef`. + The keystore file will be updated immediately. + If the issuer provided a CA certificate, a file named `truststore.p12` will + also be created in the target Secret resource, encrypted using the + password stored in `passwordSecretRef` containing the issuing Certificate + Authority type: boolean passwordSecretRef: - description: PasswordSecretRef is a reference to a key in a Secret resource containing the password used to encrypt the PKCS12 keystore. + description: |- + PasswordSecretRef is a reference to a key in a Secret resource + containing the password used to encrypt the PKCS12 keystore. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string + profile: + description: |- + Profile specifies the key and certificate encryption algorithms and the HMAC algorithm + used to create the PKCS12 keystore. Default value is `LegacyRC2` for backward compatibility. + + + If provided, allowed values are: + `LegacyRC2`: Deprecated. Not supported by default in OpenSSL 3 or Java 20. + `LegacyDES`: Less secure algorithm. Use this option for maximal compatibility. + `Modern2023`: Secure algorithm. Use this option in case you have to always use secure algorithms + (eg. because of company policy). Please note that the security of the algorithm is not that important + in reality, because the unencrypted certificate and private key are also stored in the Secret. + type: string + enum: + - LegacyRC2 + - LegacyDES + - Modern2023 literalSubject: - description: LiteralSubject is an LDAP formatted string that represents the [X.509 Subject field](https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6). Use this *instead* of the Subject field if you need to ensure the correct ordering of the RDN sequence, such as when issuing certs for LDAP authentication. See https://github.com/cert-manager/cert-manager/issues/3203, https://github.com/cert-manager/cert-manager/issues/4424. This field is alpha level and is only supported by cert-manager installations where LiteralCertificateSubject feature gate is enabled on both cert-manager controller and webhook. + description: |- + Requested X.509 certificate subject, represented using the LDAP "String + Representation of a Distinguished Name" [1]. + Important: the LDAP string format also specifies the order of the attributes + in the subject, this is important when issuing certs for LDAP authentication. + Example: `CN=foo,DC=corp,DC=example,DC=com` + More info [1]: https://datatracker.ietf.org/doc/html/rfc4514 + More info: https://github.com/cert-manager/cert-manager/issues/3203 + More info: https://github.com/cert-manager/cert-manager/issues/4424 + + + Cannot be set if the `subject` or `commonName` field is set. type: string + nameConstraints: + description: |- + x.509 certificate NameConstraint extension which MUST NOT be used in a non-CA certificate. + More Info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.10 + + + This is an Alpha Feature and is only enabled with the + `--feature-gates=NameConstraints=true` option set on both + the controller and webhook components. + type: object + properties: + critical: + description: if true then the name constraints are marked critical. + type: boolean + excluded: + description: |- + Excluded contains the constraints which must be disallowed. Any name matching a + restriction in the excluded field is invalid regardless + of information appearing in the permitted + type: object + properties: + dnsDomains: + description: DNSDomains is a list of DNS domains that are permitted or excluded. + type: array + items: + type: string + emailAddresses: + description: EmailAddresses is a list of Email Addresses that are permitted or excluded. + type: array + items: + type: string + ipRanges: + description: |- + IPRanges is a list of IP Ranges that are permitted or excluded. + This should be a valid CIDR notation. + type: array + items: + type: string + uriDomains: + description: URIDomains is a list of URI domains that are permitted or excluded. + type: array + items: + type: string + permitted: + description: Permitted contains the constraints in which the names must be located. + type: object + properties: + dnsDomains: + description: DNSDomains is a list of DNS domains that are permitted or excluded. + type: array + items: + type: string + emailAddresses: + description: EmailAddresses is a list of Email Addresses that are permitted or excluded. + type: array + items: + type: string + ipRanges: + description: |- + IPRanges is a list of IP Ranges that are permitted or excluded. + This should be a valid CIDR notation. + type: array + items: + type: string + uriDomains: + description: URIDomains is a list of URI domains that are permitted or excluded. + type: array + items: + type: string + otherNames: + description: |- + `otherNames` is an escape hatch for SAN that allows any type. We currently restrict the support to string like otherNames, cf RFC 5280 p 37 + Any UTF8 String valued otherName can be passed with by setting the keys oid: x.x.x.x and UTF8Value: somevalue for `otherName`. + Most commonly this would be UPN set with oid: 1.3.6.1.4.1.311.20.2.3 + You should ensure that any OID passed is valid for the UTF8String type as we do not explicitly validate this. + type: array + items: + type: object + properties: + oid: + description: |- + OID is the object identifier for the otherName SAN. + The object identifier must be expressed as a dotted string, for + example, "1.2.840.113556.1.4.221". + type: string + utf8Value: + description: |- + utf8Value is the string value of the otherName SAN. + The utf8Value accepts any valid UTF8 string to set as value for the otherName SAN. + type: string privateKey: - description: Options to control private keys used for the Certificate. + description: |- + Private key options. These include the key algorithm and size, the used + encoding and the rotation policy. type: object properties: algorithm: - description: Algorithm is the private key algorithm of the corresponding private key for this certificate. If provided, allowed values are either `RSA`,`Ed25519` or `ECDSA` If `algorithm` is specified and `size` is not provided, key size of 256 will be used for `ECDSA` key algorithm and key size of 2048 will be used for `RSA` key algorithm. key size is ignored when using the `Ed25519` key algorithm. + description: |- + Algorithm is the private key algorithm of the corresponding private key + for this certificate. + + + If provided, allowed values are either `RSA`, `ECDSA` or `Ed25519`. + If `algorithm` is specified and `size` is not provided, + key size of 2048 will be used for `RSA` key algorithm and + key size of 256 will be used for `ECDSA` key algorithm. + key size is ignored when using the `Ed25519` key algorithm. type: string enum: - RSA - ECDSA - Ed25519 encoding: - description: The private key cryptography standards (PKCS) encoding for this certificate's private key to be encoded in. If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 and PKCS#8, respectively. Defaults to `PKCS1` if not specified. + description: |- + The private key cryptography standards (PKCS) encoding for this + certificate's private key to be encoded in. + + + If provided, allowed values are `PKCS1` and `PKCS8` standing for PKCS#1 + and PKCS#8, respectively. + Defaults to `PKCS1` if not specified. type: string enum: - PKCS1 - PKCS8 rotationPolicy: - description: RotationPolicy controls how private keys should be regenerated when a re-issuance is being processed. If set to Never, a private key will only be generated if one does not already exist in the target `spec.secretName`. If one does exists but it does not have the correct algorithm or size, a warning will be raised to await user intervention. If set to Always, a private key matching the specified requirements will be generated whenever a re-issuance occurs. Default is 'Never' for backward compatibility. + description: |- + RotationPolicy controls how private keys should be regenerated when a + re-issuance is being processed. + + + If set to `Never`, a private key will only be generated if one does not + already exist in the target `spec.secretName`. If one does exists but it + does not have the correct algorithm or size, a warning will be raised + to await user intervention. + If set to `Always`, a private key matching the specified requirements + will be generated whenever a re-issuance occurs. + Default is `Never` for backward compatibility. type: string enum: - Never - Always size: - description: Size is the key bit size of the corresponding private key for this certificate. If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, and will default to `2048` if not specified. If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, and will default to `256` if not specified. If `algorithm` is set to `Ed25519`, Size is ignored. No other values are allowed. + description: |- + Size is the key bit size of the corresponding private key for this certificate. + + + If `algorithm` is set to `RSA`, valid values are `2048`, `4096` or `8192`, + and will default to `2048` if not specified. + If `algorithm` is set to `ECDSA`, valid values are `256`, `384` or `521`, + and will default to `256` if not specified. + If `algorithm` is set to `Ed25519`, Size is ignored. + No other values are allowed. type: integer renewBefore: - description: How long before the currently issued certificate's expiry cert-manager should renew the certificate. The default is 2/3 of the issued certificate's duration. Minimum accepted value is 5 minutes. Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration + description: |- + How long before the currently issued certificate's expiry cert-manager should + renew the certificate. For example, if a certificate is valid for 60 minutes, + and `renewBefore=10m`, cert-manager will begin to attempt to renew the certificate + 50 minutes after it was issued (i.e. when there are 10 minutes remaining until + the certificate is no longer valid). + + + NOTE: The actual lifetime of the issued certificate is used to determine the + renewal time. If an issuer returns a certificate with a different lifetime than + the one requested, cert-manager will use the lifetime of the issued certificate. + + + If unset, this defaults to 1/3 of the issued certificate's lifetime. + Minimum accepted value is 5 minutes. + Value must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration. type: string revisionHistoryLimit: - description: revisionHistoryLimit is the maximum number of CertificateRequest revisions that are maintained in the Certificate's history. Each revision represents a single `CertificateRequest` created by this Certificate, either when it was created, renewed, or Spec was changed. Revisions will be removed by oldest first if the number of revisions exceeds this number. If set, revisionHistoryLimit must be a value of `1` or greater. If unset (`nil`), revisions will not be garbage collected. Default value is `nil`. + description: |- + The maximum number of CertificateRequest revisions that are maintained in + the Certificate's history. Each revision represents a single `CertificateRequest` + created by this Certificate, either when it was created, renewed, or Spec + was changed. Revisions will be removed by oldest first if the number of + revisions exceeds this number. + + + If set, revisionHistoryLimit must be a value of `1` or greater. + If unset (`nil`), revisions will not be garbage collected. + Default value is `nil`. type: integer format: int32 secretName: - description: SecretName is the name of the secret resource that will be automatically created and managed by this Certificate resource. It will be populated with a private key and certificate, signed by the denoted issuer. + description: |- + Name of the Secret resource that will be automatically created and + managed by this Certificate resource. It will be populated with a + private key and certificate, signed by the denoted issuer. The Secret + resource lives in the same namespace as the Certificate resource. type: string secretTemplate: - description: SecretTemplate defines annotations and labels to be copied to the Certificate's Secret. Labels and annotations on the Secret will be changed as they appear on the SecretTemplate when added or removed. SecretTemplate annotations are added in conjunction with, and cannot overwrite, the base set of annotations cert-manager sets on the Certificate's Secret. + description: |- + Defines annotations and labels to be copied to the Certificate's Secret. + Labels and annotations on the Secret will be changed as they appear on the + SecretTemplate when added or removed. SecretTemplate annotations are added + in conjunction with, and cannot overwrite, the base set of annotations + cert-manager sets on the Certificate's Secret. type: object properties: annotations: @@ -423,7 +852,13 @@ spec: additionalProperties: type: string subject: - description: Full X509 name specification (https://golang.org/pkg/crypto/x509/pkix/#Name). + description: |- + Requested set of X509 certificate subject attributes. + More info: https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.6 + + + The common name attribute is specified separately in the `commonName` field. + Cannot be set if the `literalSubject` field is set. type: object properties: countries: @@ -465,15 +900,52 @@ spec: items: type: string uris: - description: URIs is a list of URI subjectAltNames to be set on the Certificate. + description: Requested URI subject alternative names. type: array items: type: string usages: - description: Usages is the set of x509 usages that are requested for the certificate. Defaults to `digital signature` and `key encipherment` if not specified. + description: |- + Requested key usages and extended key usages. + These usages are used to set the `usages` field on the created CertificateRequest + resources. If `encodeUsagesInRequest` is unset or set to `true`, the usages + will additionally be encoded in the `request` field which contains the CSR blob. + + + If unset, defaults to `digital signature` and `key encipherment`. type: array items: - description: "KeyUsage specifies valid usage contexts for keys. See: https://tools.ietf.org/html/rfc5280#section-4.2.1.3 https://tools.ietf.org/html/rfc5280#section-4.2.1.12 \n Valid KeyUsage values are as follows: \"signing\", \"digital signature\", \"content commitment\", \"key encipherment\", \"key agreement\", \"data encipherment\", \"cert sign\", \"crl sign\", \"encipher only\", \"decipher only\", \"any\", \"server auth\", \"client auth\", \"code signing\", \"email protection\", \"s/mime\", \"ipsec end system\", \"ipsec tunnel\", \"ipsec user\", \"timestamping\", \"ocsp signing\", \"microsoft sgc\", \"netscape sgc\"" + description: |- + KeyUsage specifies valid usage contexts for keys. + See: + https://tools.ietf.org/html/rfc5280#section-4.2.1.3 + https://tools.ietf.org/html/rfc5280#section-4.2.1.12 + + + Valid KeyUsage values are as follows: + "signing", + "digital signature", + "content commitment", + "key encipherment", + "key agreement", + "data encipherment", + "cert sign", + "crl sign", + "encipher only", + "decipher only", + "any", + "server auth", + "client auth", + "code signing", + "email protection", + "s/mime", + "ipsec end system", + "ipsec tunnel", + "ipsec user", + "timestamping", + "ocsp signing", + "microsoft sgc", + "netscape sgc" type: string enum: - signing @@ -500,11 +972,17 @@ spec: - microsoft sgc - netscape sgc status: - description: Status of the Certificate. This is set and managed automatically. + description: |- + Status of the Certificate. + This is set and managed automatically. + Read-only. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object properties: conditions: - description: List of status conditions to indicate the status of certificates. Known condition types are `Ready` and `Issuing`. + description: |- + List of status conditions to indicate the status of certificates. + Known condition types are `Ready` and `Issuing`. type: array items: description: CertificateCondition contains condition information for an Certificate. @@ -514,18 +992,29 @@ spec: - type properties: lastTransitionTime: - description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + description: |- + LastTransitionTime is the timestamp corresponding to the last status + change of this condition. type: string format: date-time message: - description: Message is a human readable description of the details of the last transition, complementing reason. + description: |- + Message is a human readable description of the details of the last + transition, complementing reason. type: string observedGeneration: - description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Certificate. + description: |- + If set, this represents the .metadata.generation that the condition was + set based upon. + For instance, if .metadata.generation is currently 12, but the + .status.condition[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the Certificate. type: integer format: int64 reason: - description: Reason is a brief machine readable explanation for the condition's last transition. + description: |- + Reason is a brief machine readable explanation for the condition's last + transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). @@ -541,37 +1030,85 @@ spec: - type x-kubernetes-list-type: map failedIssuanceAttempts: - description: The number of continuous failed issuance attempts up till now. This field gets removed (if set) on a successful issuance and gets set to 1 if unset and an issuance has failed. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). + description: |- + The number of continuous failed issuance attempts up till now. This + field gets removed (if set) on a successful issuance and gets set to + 1 if unset and an issuance has failed. If an issuance has failed, the + delay till the next issuance will be calculated using formula + time.Hour * 2 ^ (failedIssuanceAttempts - 1). type: integer lastFailureTime: - description: LastFailureTime is set only if the lastest issuance for this Certificate failed and contains the time of the failure. If an issuance has failed, the delay till the next issuance will be calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - 1). If the latest issuance has succeeded this field will be unset. + description: |- + LastFailureTime is set only if the lastest issuance for this + Certificate failed and contains the time of the failure. If an + issuance has failed, the delay till the next issuance will be + calculated using formula time.Hour * 2 ^ (failedIssuanceAttempts - + 1). If the latest issuance has succeeded this field will be unset. type: string format: date-time nextPrivateKeySecretName: - description: The name of the Secret resource containing the private key to be used for the next certificate iteration. The keymanager controller will automatically set this field if the `Issuing` condition is set to `True`. It will automatically unset this field when the Issuing condition is not set or False. + description: |- + The name of the Secret resource containing the private key to be used + for the next certificate iteration. + The keymanager controller will automatically set this field if the + `Issuing` condition is set to `True`. + It will automatically unset this field when the Issuing condition is + not set or False. type: string notAfter: - description: The expiration time of the certificate stored in the secret named by this resource in `spec.secretName`. + description: |- + The expiration time of the certificate stored in the secret named + by this resource in `spec.secretName`. type: string format: date-time notBefore: - description: The time after which the certificate stored in the secret named by this resource in spec.secretName is valid. + description: |- + The time after which the certificate stored in the secret named + by this resource in `spec.secretName` is valid. type: string format: date-time renewalTime: - description: RenewalTime is the time at which the certificate will be next renewed. If not set, no upcoming renewal is scheduled. + description: |- + RenewalTime is the time at which the certificate will be next + renewed. + If not set, no upcoming renewal is scheduled. type: string format: date-time revision: - description: "The current 'revision' of the certificate as issued. \n When a CertificateRequest resource is created, it will have the `cert-manager.io/certificate-revision` set to one greater than the current value of this field. \n Upon issuance, this field will be set to the value of the annotation on the CertificateRequest resource used to issue the certificate. \n Persisting the value on the CertificateRequest resource allows the certificates controller to know whether a request is part of an old issuance or if it is part of the ongoing revision's issuance by checking if the revision value in the annotation is greater than this field." + description: |- + The current 'revision' of the certificate as issued. + + + When a CertificateRequest resource is created, it will have the + `cert-manager.io/certificate-revision` set to one greater than the + current value of this field. + + + Upon issuance, this field will be set to the value of the annotation + on the CertificateRequest resource used to issue the certificate. + + + Persisting the value on the CertificateRequest resource allows the + certificates controller to know whether a request is part of an old + issuance or if it is part of the ongoing revision's issuance by + checking if the revision value in the annotation is greater than this + field. type: integer served: true storage: true + +# END crd {{- end }} + --- +# START crd {{- if or .Values.crds.enabled .Values.installCRDs }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: challenges.acme.cert-manager.io + # START annotations {{- if .Values.crds.keep }} + annotations: + helm.sh/resource-policy: keep + # END annotations {{- end }} labels: app: '{{ template "cert-manager.name" . }}' app.kubernetes.io/name: '{{ template "cert-manager.name" . }}' @@ -614,10 +1151,19 @@ spec: - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -634,13 +1180,23 @@ spec: - url properties: authorizationURL: - description: The URL to the ACME Authorization resource that this challenge is a part of. + description: |- + The URL to the ACME Authorization resource that this + challenge is a part of. type: string dnsName: - description: dnsName is the identifier that this challenge is for, e.g. example.com. If the requested DNSName is a 'wildcard', this field MUST be set to the non-wildcard domain, e.g. for `*.example.com`, it must be `example.com`. + description: |- + dnsName is the identifier that this challenge is for, e.g. example.com. + If the requested DNSName is a 'wildcard', this field MUST be set to the + non-wildcard domain, e.g. for `*.example.com`, it must be `example.com`. type: string issuerRef: - description: References a properly configured ACME-type Issuer which should be used to create this Challenge. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Challenge will be marked as failed. + description: |- + References a properly configured ACME-type Issuer which should + be used to create this Challenge. + If the Issuer does not exist, processing will be retried. + If the Issuer is not an 'ACME' Issuer, an error will be returned and the + Challenge will be marked as failed. type: object required: - name @@ -655,34 +1211,54 @@ spec: description: Name of the resource being referred to. type: string key: - description: 'The ACME challenge key for this challenge For HTTP01 challenges, this is the value that must be responded with to complete the HTTP01 challenge in the format: `.`. For DNS01 challenges, this is the base64 encoded SHA256 sum of the `.` text that must be set as the TXT record content.' + description: |- + The ACME challenge key for this challenge + For HTTP01 challenges, this is the value that must be responded with to + complete the HTTP01 challenge in the format: + `.`. + For DNS01 challenges, this is the base64 encoded SHA256 sum of the + `.` + text that must be set as the TXT record content. type: string solver: - description: Contains the domain solving configuration that should be used to solve this challenge resource. + description: |- + Contains the domain solving configuration that should be used to + solve this challenge resource. type: object properties: dns01: - description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. + description: |- + Configures cert-manager to attempt to complete authorizations by + performing the DNS01 challenge flow. type: object properties: acmeDNS: - description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. + description: |- + Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage + DNS01 challenge records. type: object required: - accountSecretRef - host properties: accountSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string host: type: string @@ -696,40 +1272,61 @@ spec: - serviceConsumerDomain properties: accessTokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientSecretSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientTokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceConsumerDomain: type: string @@ -741,19 +1338,30 @@ spec: - subscriptionID properties: clientID: - description: if both this and ClientSecret are left unset MSI will be used + description: |- + Auth: Azure Service Principal: + The ClientID of the Azure Service Principal used to authenticate with Azure DNS. + If set, ClientSecret and TenantID must also be set. type: string clientSecretSecretRef: - description: if both this and ClientID are left unset MSI will be used + description: |- + Auth: Azure Service Principal: + A reference to a Secret containing the password associated with the Service Principal. + If set, ClientID and TenantID must also be set. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string environment: description: name of the Azure environment (default AzurePublicCloud) @@ -767,14 +1375,19 @@ spec: description: name of the DNS zone that should be used type: string managedIdentity: - description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID + description: |- + Auth: Azure Workload Identity or Azure Managed Service Identity: + Settings to enable Azure Workload Identity or Azure Managed Service Identity + If set, ClientID, ClientSecret and TenantID must not be set. type: object properties: clientID: description: client ID of the managed identity, can not be used at the same time as resourceID type: string resourceID: - description: resource ID of the managed identity, can not be used at the same time as clientID + description: |- + resource ID of the managed identity, can not be used at the same time as clientID + Cannot be used for Azure Managed Service Identity type: string resourceGroupName: description: resource group the DNS zone is located in @@ -783,7 +1396,10 @@ spec: description: ID of the Azure subscription type: string tenantID: - description: when specifying ClientID and ClientSecret then this field is also needed + description: |- + Auth: Azure Service Principal: + The TenantID of the Azure Service Principal used to authenticate with Azure DNS. + If set, ClientID and ClientSecret must also be set. type: string cloudDNS: description: Use the Google Cloud DNS API to manage DNS01 challenge records. @@ -792,37 +1408,55 @@ spec: - project properties: hostedZoneName: - description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. + description: |- + HostedZoneName is an optional field that tells cert-manager in which + Cloud DNS zone the challenge record has to be created. + If left empty cert-manager will automatically choose a zone. type: string project: type: string serviceAccountSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string cloudflare: description: Use the Cloudflare API to manage DNS01 challenge records. type: object properties: apiKeySecretRef: - description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.' + description: |- + API key to use to authenticate with Cloudflare. + Note: using an API token to authenticate is now the recommended method + as it allows greater control of permissions. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string apiTokenSecretRef: description: API token used to authenticate with Cloudflare. @@ -831,16 +1465,23 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string email: description: Email of the account, only required when using API key based authentication. type: string cnameStrategy: - description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. + description: |- + CNAMEStrategy configures how the DNS01 provider should handle CNAME + records when found in DNS zones. type: string enum: - None @@ -852,43 +1493,69 @@ spec: - tokenSecretRef properties: tokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string rfc2136: - description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. + description: |- + Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) + to manage DNS01 challenge records. type: object required: - nameserver properties: nameserver: - description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + description: |- + The IP address or hostname of an authoritative DNS server supporting + RFC2136 in the form host:port. If the host is an IPv6 address it must be + enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. + This field is required. type: string tsigAlgorithm: - description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' + description: |- + The TSIG Algorithm configured in the DNS supporting RFC2136. Used only + when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. + Supported values are (case-insensitive): ``HMACMD5`` (default), + ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``. type: string tsigKeyName: - description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. + description: |- + The TSIG Key name configured in the DNS. + If ``tsigSecretSecretRef`` is defined, this field is required. type: string tsigSecretSecretRef: - description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. + description: |- + The name of the secret containing the TSIG value. + If ``tsigKeyName`` is defined, this field is required. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string route53: description: Use the AWS Route53 API to manage DNS01 challenge records. @@ -897,20 +1564,71 @@ spec: - region properties: accessKeyID: - description: 'The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The AccessKeyID is used for authentication. + Cannot be set when SecretAccessKeyID is set. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: string accessKeyIDSecretRef: - description: 'The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The SecretAccessKey is used for authentication. If set, pull the AWS + access key ID from a key within a Kubernetes Secret. + Cannot be set when AccessKeyID is set. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string + auth: + description: Auth configures how cert-manager authenticates. + type: object + required: + - kubernetes + properties: + kubernetes: + description: |- + Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity + by passing a bound ServiceAccount token. + type: object + required: + - serviceAccountRef + properties: + serviceAccountRef: + description: |- + A reference to a service account that will be used to request a bound + token (also known as "projected token"). To use this field, you must + configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + audiences: + description: |- + TokenAudiences is an optional list of audiences to include in the + token passed to AWS. The default token consisting of the issuer's namespace + and name is always included. + If unset the audience defaults to `sts.amazonaws.com`. + type: array + items: + type: string + name: + description: Name of the ServiceAccount used to request a token. + type: string hostedZoneID: description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. type: string @@ -918,113 +1636,301 @@ spec: description: Always set the region when using AccessKeyID and SecretAccessKey type: string role: - description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata + description: |- + Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey + or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata type: string secretAccessKeySecretRef: - description: 'The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The SecretAccessKey is used for authentication. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string webhook: - description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. + description: |- + Configure an external webhook based DNS01 challenge solver to manage + DNS01 challenge records. type: object required: - groupName - solverName properties: config: - description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. + description: |- + Additional configuration that should be passed to the webhook apiserver + when challenges are processed. + This can contain arbitrary JSON data. + Secret values should not be specified in this stanza. + If secret values are needed (e.g. credentials for a DNS service), you + should use a SecretKeySelector to reference a Secret resource. + For details on the schema of this field, consult the webhook provider + implementation's documentation. x-kubernetes-preserve-unknown-fields: true groupName: - description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. + description: |- + The API group name that should be used when POSTing ChallengePayload + resources to the webhook apiserver. + This should be the same as the GroupName specified in the webhook + provider implementation. type: string solverName: - description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. + description: |- + The name of the solver to use, as defined in the webhook provider + implementation. + This will typically be the name of the provider, e.g. 'cloudflare'. type: string http01: - description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. + description: |- + Configures cert-manager to attempt to complete authorizations by + performing the HTTP01 challenge flow. + It is not possible to obtain certificates for wildcard domain names + (e.g. `*.example.com`) using the HTTP01 challenge mechanism. type: object properties: gatewayHTTPRoute: - description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. + description: |- + The Gateway API is a sig-network community API that models service networking + in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will + create HTTPRoutes with the specified labels in the same namespace as the challenge. + This solver is experimental, and fields / behaviour may change in the future. type: object properties: labels: - description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. + description: |- + Custom labels that will be applied to HTTPRoutes created by cert-manager + while solving HTTP-01 challenges. type: object additionalProperties: type: string parentRefs: - description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' + description: |- + When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. + cert-manager needs to know which parentRefs should be used when creating + the HTTPRoute. Usually, the parentRef references a Gateway. See: + https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways type: array items: - description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + + This API may be extended in the future to support additional kinds of parent + resources. + + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. type: object required: - name properties: group: - description: "Group is the group of the referent. When unspecified, \"gateway.networking.k8s.io\" is inferred. To set the core API group (such as for a \"Service\" kind referent), Group must be explicitly set to \"\" (empty string). \n Support: Core" + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + + Support: Core type: string default: gateway.networking.k8s.io maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: - description: "Kind is kind of the referent. \n Support: Core (Gateway) \n Support: Implementation-specific (Other Resources)" + description: |- + Kind is kind of the referent. + + + There are two kinds of parent resources with "Core" support: + + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + + Support for other resources is Implementation-Specific. type: string default: Gateway maxLength: 63 minLength: 1 pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ name: - description: "Name is the name of the referent. \n Support: Core" + description: |- + Name is the name of the referent. + + + Support: Core type: string maxLength: 253 minLength: 1 namespace: - description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n Support: Core" + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + + Support: Core type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: - description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + + Support: Extended type: integer format: int32 maximum: 65535 minimum: 1 sectionName: - description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + + Support: Core type: string maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ serviceType: - description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + description: |- + Optional service type for Kubernetes solver service. Supported values + are NodePort or ClusterIP. If unset, defaults to NodePort. type: string ingress: - description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. + description: |- + The ingress based HTTP01 challenge solver will solve challenges by + creating or modifying Ingress resources in order to route requests for + '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are + provisioned by cert-manager for each Challenge to be completed. type: object properties: class: - description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + This field configures the annotation `kubernetes.io/ingress.class` when + creating Ingress resources to solve ACME challenges that use this + challenge solver. Only one of `class`, `name` or `ingressClassName` may + be specified. type: string ingressClassName: - description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + This field configures the field `ingressClassName` on the created Ingress + resources used to solve ACME challenges that use this challenge solver. + This is the recommended way of configuring the ingress class. Only one of + `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: - description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. + description: |- + Optional ingress template used to configure the ACME challenge solver + ingress used for HTTP01 challenges. type: object properties: metadata: - description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + description: |- + ObjectMeta overrides for the ingress used to solve HTTP01 challenges. + Only the 'labels' and 'annotations' fields may be set. + If labels or annotations overlap with in-built values, the values here + will override the in-built values. type: object properties: annotations: @@ -1038,14 +1944,26 @@ spec: additionalProperties: type: string name: - description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + The name of the ingress resource that should have ACME challenge solving + routes inserted into it in order to solve HTTP01 challenges. + This is typically used in conjunction with ingress controllers like + ingress-gce, which maintains a 1:1 mapping between external IPs and + ingress resources. Only one of `class`, `name` or `ingressClassName` may + be specified. type: string podTemplate: - description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. + description: |- + Optional pod template used to configure the ACME challenge solver pods + used for HTTP01 challenges. type: object properties: metadata: - description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + description: |- + ObjectMeta overrides for the pod used to solve HTTP01 challenges. + Only the 'labels' and 'annotations' fields may be set. + If labels or annotations overlap with in-built values, the values here + will override the in-built values. type: object properties: annotations: @@ -1059,7 +1977,10 @@ spec: additionalProperties: type: string spec: - description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. + description: |- + PodSpec defines overrides for the HTTP01 challenge solver pod. + Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. + All other fields will be ignored. type: object properties: affinity: @@ -1071,10 +1992,21 @@ spec: type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. type: array items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). type: object required: - preference @@ -1088,7 +2020,9 @@ spec: description: A list of node selector requirements by node's labels. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -1098,18 +2032,29 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -1119,20 +2064,35 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. type: object required: - nodeSelectorTerms @@ -1141,14 +2101,19 @@ spec: description: Required. A list of node selector terms. The terms are ORed. type: array items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -1158,18 +2123,29 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -1179,21 +2155,40 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -1209,14 +2204,18 @@ spec: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1226,28 +2225,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1257,49 +2304,90 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1309,28 +2397,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1340,33 +2476,64 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string + x-kubernetes-list-type: atomic podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -1382,14 +2549,18 @@ spec: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1399,28 +2570,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1430,49 +2649,90 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1482,28 +2742,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -1513,40 +2821,75 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string + x-kubernetes-list-type: atomic imagePullSecrets: description: If specified, the pod's imagePullSecrets type: array items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. type: object properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string + default: "" x-kubernetes-map-type: atomic nodeSelector: - description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object additionalProperties: type: string @@ -1560,76 +2903,141 @@ spec: description: If specified, the pod's tolerations. type: array items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . type: object properties: effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. type: string tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. type: integer format: int64 value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. type: string serviceType: - description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + description: |- + Optional service type for Kubernetes solver service. Supported values + are NodePort or ClusterIP. If unset, defaults to NodePort. type: string selector: - description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. + description: |- + Selector selects a set of DNSNames on the Certificate resource that + should be solved using this challenge solver. + If not specified, the solver will be treated as the 'default' solver + with the lowest priority, i.e. if any other solver has a more specific + match, it will be used instead. type: object properties: dnsNames: - description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + description: |- + List of DNSNames that this solver will be used to solve. + If specified and a match is found, a dnsNames selector will take + precedence over a dnsZones selector. + If multiple solvers match with the same dnsNames value, the solver + with the most matching labels in matchLabels will be selected. + If neither has more matches, the solver defined earlier in the list + will be selected. type: array items: type: string dnsZones: - description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + description: |- + List of DNSZones that this solver will be used to solve. + The most specific DNS zone match specified here will take precedence + over other DNS zone matches, so a solver specifying sys.example.com + will be selected over one specifying example.com for the domain + www.sys.example.com. + If multiple solvers match with the same dnsZones value, the solver + with the most matching labels in matchLabels will be selected. + If neither has more matches, the solver defined earlier in the list + will be selected. type: array items: type: string matchLabels: - description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to. + description: |- + A label selector that is used to refine the set of certificate's that + this challenge solver will apply to. type: object additionalProperties: type: string token: - description: The ACME challenge token for this challenge. This is the raw value returned from the ACME server. + description: |- + The ACME challenge token for this challenge. + This is the raw value returned from the ACME server. type: string type: - description: The type of ACME challenge this resource represents. One of "HTTP-01" or "DNS-01". + description: |- + The type of ACME challenge this resource represents. + One of "HTTP-01" or "DNS-01". type: string enum: - HTTP-01 - DNS-01 url: - description: The URL of the ACME Challenge resource for this challenge. This can be used to lookup details about the status of this challenge. + description: |- + The URL of the ACME Challenge resource for this challenge. + This can be used to lookup details about the status of this challenge. type: string wildcard: - description: wildcard will be true if this challenge is for a wildcard identifier, for example '*.example.com'. + description: |- + wildcard will be true if this challenge is for a wildcard identifier, + for example '*.example.com'. type: boolean status: type: object properties: presented: - description: presented will be set to true if the challenge values for this challenge are currently 'presented'. This *does not* imply the self check is passing. Only that the values have been 'submitted' for the appropriate challenge mechanism (i.e. the DNS01 TXT record has been presented, or the HTTP01 configuration has been configured). + description: |- + presented will be set to true if the challenge values for this challenge + are currently 'presented'. + This *does not* imply the self check is passing. Only that the values + have been 'submitted' for the appropriate challenge mechanism (i.e. the + DNS01 TXT record has been presented, or the HTTP01 configuration has been + configured). type: boolean processing: - description: Used to denote whether this challenge should be processed or not. This field will only be set to true by the 'scheduling' component. It will only be set to false by the 'challenges' controller, after the challenge has reached a final state or timed out. If this field is set to false, the challenge controller will not take any more action. + description: |- + Used to denote whether this challenge should be processed or not. + This field will only be set to true by the 'scheduling' component. + It will only be set to false by the 'challenges' controller, after the + challenge has reached a final state or timed out. + If this field is set to false, the challenge controller will not take + any more action. type: boolean reason: - description: Contains human readable information on why the Challenge is in the current state. + description: |- + Contains human readable information on why the Challenge is in the + current state. type: string state: - description: Contains the current 'state' of the challenge. If not set, the state of the challenge is unknown. + description: |- + Contains the current 'state' of the challenge. + If not set, the state of the challenge is unknown. type: string enum: - valid @@ -1643,15 +3051,23 @@ spec: storage: true subresources: status: {} + +# END crd {{- end }} + --- +# START crd {{- if or .Values.crds.enabled .Values.installCRDs }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clusterissuers.cert-manager.io + # START annotations {{- if .Values.crds.keep }} + annotations: + helm.sh/resource-policy: keep + # END annotations {{- end }} labels: app: '{{ template "cert-manager.name" . }}' app.kubernetes.io/name: '{{ template "cert-manager.name" . }}' - app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/instance: '{{ .Release.Name }}' # Generated labels {{- include "labels" . | nindent 4 }} spec: group: cert-manager.io @@ -1681,16 +3097,30 @@ spec: type: date schema: openAPIV3Schema: - description: A ClusterIssuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is similar to an Issuer, however it is cluster-scoped and therefore can be referenced by resources that exist in *any* namespace, not just the same namespace as the referent. + description: |- + A ClusterIssuer represents a certificate issuing authority which can be + referenced as part of `issuerRef` fields. + It is similar to an Issuer, however it is cluster-scoped and therefore can + be referenced by resources that exist in *any* namespace, not just the same + namespace as the referent. type: object required: - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -1699,34 +3129,65 @@ spec: type: object properties: acme: - description: ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates. + description: |- + ACME configures this issuer to communicate with a RFC8555 (ACME) server + to obtain signed x509 certificates. type: object required: - privateKeySecretRef - server properties: caBundle: - description: Base64-encoded bundle of PEM CAs which can be used to validate the certificate chain presented by the ACME server. Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various kinds of security vulnerabilities. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. + description: |- + Base64-encoded bundle of PEM CAs which can be used to validate the certificate + chain presented by the ACME server. + Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various + kinds of security vulnerabilities. + If CABundle and SkipTLSVerify are unset, the system certificate bundle inside + the container is used to validate the TLS connection. type: string format: byte disableAccountKeyGeneration: - description: Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false. + description: |- + Enables or disables generating a new ACME account key. + If true, the Issuer resource will *not* request a new account but will expect + the account key to be supplied via an existing secret. + If false, the cert-manager system will generate a new ACME account key + for the Issuer. + Defaults to false. type: boolean email: - description: Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered. + description: |- + Email is the email address to be associated with the ACME account. + This field is optional, but it is strongly recommended to be set. + It will be used to contact you in case of issues with your account or + certificates, including expiry notification emails. + This field may be updated after the account is initially registered. type: string enableDurationFeature: - description: Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it it will create an error on the Order. Defaults to false. + description: |- + Enables requesting a Not After date on certificates that matches the + duration of the certificate. This is not supported by all ACME servers + like Let's Encrypt. If set to true when the ACME server does not support + it, it will create an error on the Order. + Defaults to false. type: boolean externalAccountBinding: - description: ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account. + description: |- + ExternalAccountBinding is a reference to a CA external account of the ACME + server. + If set, upon registration cert-manager will attempt to associate the given + external account credentials with the registered ACME account. type: object required: - keyID - keySecretRef properties: keyAlgorithm: - description: 'Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme.' + description: |- + Deprecated: keyAlgorithm field exists for historical compatibility + reasons and should not be used. The algorithm is now hardcoded to HS256 + in golang/x/crypto/acme. type: string enum: - HS256 @@ -1736,68 +3197,130 @@ spec: description: keyID is the ID of the CA key that the External Account is bound to. type: string keySecretRef: - description: keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data. + description: |- + keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes + Secret which holds the symmetric MAC key of the External Account Binding. + The `key` is the index string that is paired with the key data in the + Secret and should not be confused with the key data itself, or indeed with + the External Account Binding keyID above. + The secret key stored in the Secret **must** be un-padded, base64 URL + encoded data. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string preferredChain: - description: 'PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let''s Encrypt''s DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let''s Encrypt root CA. This value picks the first certificate bundle in the ACME alternative chains that has a certificate with this value as its issuer''s CN' + description: |- + PreferredChain is the chain to use if the ACME server outputs multiple. + PreferredChain is no guarantee that this one gets delivered by the ACME + endpoint. + For example, for Let's Encrypt's DST crosssign you would use: + "DST Root CA X3" or "ISRG Root X1" for the newer Let's Encrypt root CA. + This value picks the first certificate bundle in the combined set of + ACME default and alternative chains that has a root-most certificate with + this value as its issuer's commonname. type: string maxLength: 64 privateKeySecretRef: - description: PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used. + description: |- + PrivateKey is the name of a Kubernetes Secret resource that will be used to + store the automatically generated ACME account private key. + Optionally, a `key` may be specified to select a specific entry within + the named Secret resource. + If `key` is not specified, a default of `tls.key` will be used. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string server: - description: 'Server is the URL used to access the ACME server''s ''directory'' endpoint. For example, for Let''s Encrypt''s staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported.' + description: |- + Server is the URL used to access the ACME server's 'directory' endpoint. + For example, for Let's Encrypt's staging endpoint, you would use: + "https://acme-staging-v02.api.letsencrypt.org/directory". + Only ACME v2 endpoints (i.e. RFC 8555) are supported. type: string skipTLSVerify: - description: 'INSECURE: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have the TLS certificate chain validated. Mutually exclusive with CABundle; prefer using CABundle to prevent various kinds of security vulnerabilities. Only enable this option in development environments. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. Defaults to false.' + description: |- + INSECURE: Enables or disables validation of the ACME server TLS certificate. + If true, requests to the ACME server will not have the TLS certificate chain + validated. + Mutually exclusive with CABundle; prefer using CABundle to prevent various + kinds of security vulnerabilities. + Only enable this option in development environments. + If CABundle and SkipTLSVerify are unset, the system certificate bundle inside + the container is used to validate the TLS connection. + Defaults to false. type: boolean solvers: - description: 'Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/' + description: |- + Solvers is a list of challenge solvers that will be used to solve + ACME challenges for the matching domains. + Solver configurations must be provided in order to obtain certificates + from an ACME server. + For more information, see: https://cert-manager.io/docs/configuration/acme/ type: array items: - description: An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided. + description: |- + An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. + A selector may be provided to use different solving strategies for different DNS names. + Only one of HTTP01 or DNS01 must be provided. type: object properties: dns01: - description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. + description: |- + Configures cert-manager to attempt to complete authorizations by + performing the DNS01 challenge flow. type: object properties: acmeDNS: - description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. + description: |- + Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage + DNS01 challenge records. type: object required: - accountSecretRef - host properties: accountSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string host: type: string @@ -1811,40 +3334,61 @@ spec: - serviceConsumerDomain properties: accessTokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientSecretSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientTokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceConsumerDomain: type: string @@ -1856,19 +3400,30 @@ spec: - subscriptionID properties: clientID: - description: if both this and ClientSecret are left unset MSI will be used + description: |- + Auth: Azure Service Principal: + The ClientID of the Azure Service Principal used to authenticate with Azure DNS. + If set, ClientSecret and TenantID must also be set. type: string clientSecretSecretRef: - description: if both this and ClientID are left unset MSI will be used + description: |- + Auth: Azure Service Principal: + A reference to a Secret containing the password associated with the Service Principal. + If set, ClientID and TenantID must also be set. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string environment: description: name of the Azure environment (default AzurePublicCloud) @@ -1882,14 +3437,19 @@ spec: description: name of the DNS zone that should be used type: string managedIdentity: - description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID + description: |- + Auth: Azure Workload Identity or Azure Managed Service Identity: + Settings to enable Azure Workload Identity or Azure Managed Service Identity + If set, ClientID, ClientSecret and TenantID must not be set. type: object properties: clientID: description: client ID of the managed identity, can not be used at the same time as resourceID type: string resourceID: - description: resource ID of the managed identity, can not be used at the same time as clientID + description: |- + resource ID of the managed identity, can not be used at the same time as clientID + Cannot be used for Azure Managed Service Identity type: string resourceGroupName: description: resource group the DNS zone is located in @@ -1898,7 +3458,10 @@ spec: description: ID of the Azure subscription type: string tenantID: - description: when specifying ClientID and ClientSecret then this field is also needed + description: |- + Auth: Azure Service Principal: + The TenantID of the Azure Service Principal used to authenticate with Azure DNS. + If set, ClientID and ClientSecret must also be set. type: string cloudDNS: description: Use the Google Cloud DNS API to manage DNS01 challenge records. @@ -1907,37 +3470,55 @@ spec: - project properties: hostedZoneName: - description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. + description: |- + HostedZoneName is an optional field that tells cert-manager in which + Cloud DNS zone the challenge record has to be created. + If left empty cert-manager will automatically choose a zone. type: string project: type: string serviceAccountSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string cloudflare: description: Use the Cloudflare API to manage DNS01 challenge records. type: object properties: apiKeySecretRef: - description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.' + description: |- + API key to use to authenticate with Cloudflare. + Note: using an API token to authenticate is now the recommended method + as it allows greater control of permissions. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string apiTokenSecretRef: description: API token used to authenticate with Cloudflare. @@ -1946,16 +3527,23 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string email: description: Email of the account, only required when using API key based authentication. type: string cnameStrategy: - description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. + description: |- + CNAMEStrategy configures how the DNS01 provider should handle CNAME + records when found in DNS zones. type: string enum: - None @@ -1967,43 +3555,69 @@ spec: - tokenSecretRef properties: tokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string rfc2136: - description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. + description: |- + Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) + to manage DNS01 challenge records. type: object required: - nameserver properties: nameserver: - description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + description: |- + The IP address or hostname of an authoritative DNS server supporting + RFC2136 in the form host:port. If the host is an IPv6 address it must be + enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. + This field is required. type: string tsigAlgorithm: - description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' + description: |- + The TSIG Algorithm configured in the DNS supporting RFC2136. Used only + when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. + Supported values are (case-insensitive): ``HMACMD5`` (default), + ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``. type: string tsigKeyName: - description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. + description: |- + The TSIG Key name configured in the DNS. + If ``tsigSecretSecretRef`` is defined, this field is required. type: string tsigSecretSecretRef: - description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. + description: |- + The name of the secret containing the TSIG value. + If ``tsigKeyName`` is defined, this field is required. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string route53: description: Use the AWS Route53 API to manage DNS01 challenge records. @@ -2012,20 +3626,71 @@ spec: - region properties: accessKeyID: - description: 'The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The AccessKeyID is used for authentication. + Cannot be set when SecretAccessKeyID is set. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: string accessKeyIDSecretRef: - description: 'The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The SecretAccessKey is used for authentication. If set, pull the AWS + access key ID from a key within a Kubernetes Secret. + Cannot be set when AccessKeyID is set. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string + auth: + description: Auth configures how cert-manager authenticates. + type: object + required: + - kubernetes + properties: + kubernetes: + description: |- + Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity + by passing a bound ServiceAccount token. + type: object + required: + - serviceAccountRef + properties: + serviceAccountRef: + description: |- + A reference to a service account that will be used to request a bound + token (also known as "projected token"). To use this field, you must + configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + audiences: + description: |- + TokenAudiences is an optional list of audiences to include in the + token passed to AWS. The default token consisting of the issuer's namespace + and name is always included. + If unset the audience defaults to `sts.amazonaws.com`. + type: array + items: + type: string + name: + description: Name of the ServiceAccount used to request a token. + type: string hostedZoneID: description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. type: string @@ -2033,113 +3698,301 @@ spec: description: Always set the region when using AccessKeyID and SecretAccessKey type: string role: - description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata + description: |- + Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey + or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata type: string secretAccessKeySecretRef: - description: 'The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The SecretAccessKey is used for authentication. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string webhook: - description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. + description: |- + Configure an external webhook based DNS01 challenge solver to manage + DNS01 challenge records. type: object required: - groupName - solverName properties: config: - description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. + description: |- + Additional configuration that should be passed to the webhook apiserver + when challenges are processed. + This can contain arbitrary JSON data. + Secret values should not be specified in this stanza. + If secret values are needed (e.g. credentials for a DNS service), you + should use a SecretKeySelector to reference a Secret resource. + For details on the schema of this field, consult the webhook provider + implementation's documentation. x-kubernetes-preserve-unknown-fields: true groupName: - description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. + description: |- + The API group name that should be used when POSTing ChallengePayload + resources to the webhook apiserver. + This should be the same as the GroupName specified in the webhook + provider implementation. type: string solverName: - description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. + description: |- + The name of the solver to use, as defined in the webhook provider + implementation. + This will typically be the name of the provider, e.g. 'cloudflare'. type: string http01: - description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. + description: |- + Configures cert-manager to attempt to complete authorizations by + performing the HTTP01 challenge flow. + It is not possible to obtain certificates for wildcard domain names + (e.g. `*.example.com`) using the HTTP01 challenge mechanism. type: object properties: gatewayHTTPRoute: - description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. + description: |- + The Gateway API is a sig-network community API that models service networking + in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will + create HTTPRoutes with the specified labels in the same namespace as the challenge. + This solver is experimental, and fields / behaviour may change in the future. type: object properties: labels: - description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. + description: |- + Custom labels that will be applied to HTTPRoutes created by cert-manager + while solving HTTP-01 challenges. type: object additionalProperties: type: string parentRefs: - description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' + description: |- + When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. + cert-manager needs to know which parentRefs should be used when creating + the HTTPRoute. Usually, the parentRef references a Gateway. See: + https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways type: array items: - description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + + This API may be extended in the future to support additional kinds of parent + resources. + + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. type: object required: - name properties: group: - description: "Group is the group of the referent. When unspecified, \"gateway.networking.k8s.io\" is inferred. To set the core API group (such as for a \"Service\" kind referent), Group must be explicitly set to \"\" (empty string). \n Support: Core" + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + + Support: Core type: string default: gateway.networking.k8s.io maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: - description: "Kind is kind of the referent. \n Support: Core (Gateway) \n Support: Implementation-specific (Other Resources)" + description: |- + Kind is kind of the referent. + + + There are two kinds of parent resources with "Core" support: + + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + + Support for other resources is Implementation-Specific. type: string default: Gateway maxLength: 63 minLength: 1 pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ name: - description: "Name is the name of the referent. \n Support: Core" + description: |- + Name is the name of the referent. + + + Support: Core type: string maxLength: 253 minLength: 1 namespace: - description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n Support: Core" + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + + Support: Core type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: - description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + + Support: Extended type: integer format: int32 maximum: 65535 minimum: 1 sectionName: - description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + + Support: Core type: string maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ serviceType: - description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + description: |- + Optional service type for Kubernetes solver service. Supported values + are NodePort or ClusterIP. If unset, defaults to NodePort. type: string ingress: - description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. + description: |- + The ingress based HTTP01 challenge solver will solve challenges by + creating or modifying Ingress resources in order to route requests for + '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are + provisioned by cert-manager for each Challenge to be completed. type: object properties: class: - description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + This field configures the annotation `kubernetes.io/ingress.class` when + creating Ingress resources to solve ACME challenges that use this + challenge solver. Only one of `class`, `name` or `ingressClassName` may + be specified. type: string ingressClassName: - description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + This field configures the field `ingressClassName` on the created Ingress + resources used to solve ACME challenges that use this challenge solver. + This is the recommended way of configuring the ingress class. Only one of + `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: - description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. + description: |- + Optional ingress template used to configure the ACME challenge solver + ingress used for HTTP01 challenges. type: object properties: metadata: - description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + description: |- + ObjectMeta overrides for the ingress used to solve HTTP01 challenges. + Only the 'labels' and 'annotations' fields may be set. + If labels or annotations overlap with in-built values, the values here + will override the in-built values. type: object properties: annotations: @@ -2153,14 +4006,26 @@ spec: additionalProperties: type: string name: - description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + The name of the ingress resource that should have ACME challenge solving + routes inserted into it in order to solve HTTP01 challenges. + This is typically used in conjunction with ingress controllers like + ingress-gce, which maintains a 1:1 mapping between external IPs and + ingress resources. Only one of `class`, `name` or `ingressClassName` may + be specified. type: string podTemplate: - description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. + description: |- + Optional pod template used to configure the ACME challenge solver pods + used for HTTP01 challenges. type: object properties: metadata: - description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + description: |- + ObjectMeta overrides for the pod used to solve HTTP01 challenges. + Only the 'labels' and 'annotations' fields may be set. + If labels or annotations overlap with in-built values, the values here + will override the in-built values. type: object properties: annotations: @@ -2174,7 +4039,10 @@ spec: additionalProperties: type: string spec: - description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. + description: |- + PodSpec defines overrides for the HTTP01 challenge solver pod. + Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. + All other fields will be ignored. type: object properties: affinity: @@ -2186,10 +4054,21 @@ spec: type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. type: array items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). type: object required: - preference @@ -2203,7 +4082,9 @@ spec: description: A list of node selector requirements by node's labels. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -2213,18 +4094,29 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -2234,20 +4126,35 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. type: object required: - nodeSelectorTerms @@ -2256,14 +4163,19 @@ spec: description: Required. A list of node selector terms. The terms are ORed. type: array items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -2273,18 +4185,29 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -2294,21 +4217,40 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -2324,14 +4266,18 @@ spec: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2341,28 +4287,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2372,49 +4366,90 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2424,28 +4459,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2455,33 +4538,64 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string + x-kubernetes-list-type: atomic podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -2497,14 +4611,18 @@ spec: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2514,28 +4632,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2545,49 +4711,90 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2597,28 +4804,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -2628,40 +4883,75 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string + x-kubernetes-list-type: atomic imagePullSecrets: description: If specified, the pod's imagePullSecrets type: array items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. type: object properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string + default: "" x-kubernetes-map-type: atomic nodeSelector: - description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object additionalProperties: type: string @@ -2675,77 +4965,146 @@ spec: description: If specified, the pod's tolerations. type: array items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . type: object properties: effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. type: string tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. type: integer format: int64 value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. type: string serviceType: - description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + description: |- + Optional service type for Kubernetes solver service. Supported values + are NodePort or ClusterIP. If unset, defaults to NodePort. type: string selector: - description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. + description: |- + Selector selects a set of DNSNames on the Certificate resource that + should be solved using this challenge solver. + If not specified, the solver will be treated as the 'default' solver + with the lowest priority, i.e. if any other solver has a more specific + match, it will be used instead. type: object properties: dnsNames: - description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + description: |- + List of DNSNames that this solver will be used to solve. + If specified and a match is found, a dnsNames selector will take + precedence over a dnsZones selector. + If multiple solvers match with the same dnsNames value, the solver + with the most matching labels in matchLabels will be selected. + If neither has more matches, the solver defined earlier in the list + will be selected. type: array items: type: string dnsZones: - description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + description: |- + List of DNSZones that this solver will be used to solve. + The most specific DNS zone match specified here will take precedence + over other DNS zone matches, so a solver specifying sys.example.com + will be selected over one specifying example.com for the domain + www.sys.example.com. + If multiple solvers match with the same dnsZones value, the solver + with the most matching labels in matchLabels will be selected. + If neither has more matches, the solver defined earlier in the list + will be selected. type: array items: type: string matchLabels: - description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to. + description: |- + A label selector that is used to refine the set of certificate's that + this challenge solver will apply to. type: object additionalProperties: type: string ca: - description: CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager. + description: |- + CA configures this issuer to sign certificates using a signing CA keypair + stored in a Secret resource. + This is used to build internal PKIs that are managed by cert-manager. type: object required: - secretName properties: crlDistributionPoints: - description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set. + description: |- + The CRL distribution points is an X.509 v3 certificate extension which identifies + the location of the CRL from which the revocation of this certificate can be checked. + If not set, certificates will be issued without distribution points set. + type: array + items: + type: string + issuingCertificateURLs: + description: |- + IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates + it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details. + As an example, such a URL might be "http://ca.domain.com/ca.crt". type: array items: type: string ocspServers: - description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". + description: |- + The OCSP server list is an X.509 v3 extension that defines a list of + URLs of OCSP responders. The OCSP responders can be queried for the + revocation status of an issued certificate. If not set, the + certificate will be issued with no OCSP servers set. For example, an + OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". type: array items: type: string secretName: - description: SecretName is the name of the secret used to sign Certificates issued by this Issuer. + description: |- + SecretName is the name of the secret used to sign Certificates issued + by this Issuer. type: string selfSigned: - description: SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object. + description: |- + SelfSigned configures this issuer to 'self sign' certificates using the + private key used to create the CertificateRequest object. type: object properties: crlDistributionPoints: - description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings. + description: |- + The CRL distribution points is an X.509 v3 certificate extension which identifies + the location of the CRL from which the revocation of this certificate can be checked. + If not set certificate will be issued without CDP. Values are strings. type: array items: type: string vault: - description: Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend. + description: |- + Vault configures this issuer to sign certificates using a HashiCorp Vault + PKI backend. type: object required: - auth @@ -2757,7 +5116,9 @@ spec: type: object properties: appRole: - description: AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource. + description: |- + AppRole authenticates with Vault using the App Role auth mechanism, + with the role and secret stored in a Kubernetes Secret resource. type: object required: - path @@ -2765,53 +5126,94 @@ spec: - secretRef properties: path: - description: 'Path where the App Role authentication backend is mounted in Vault, e.g: "approle"' + description: |- + Path where the App Role authentication backend is mounted in Vault, e.g: + "approle" type: string roleId: - description: RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault. + description: |- + RoleID configured in the App Role authentication backend when setting + up the authentication backend in Vault. type: string secretRef: - description: Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret. + description: |- + Reference to a key in a Secret that contains the App Role secret used + to authenticate with Vault. + The `key` field must be specified and denotes which entry within the Secret + resource is used as the app role secret. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string kubernetes: - description: Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server. + description: |- + Kubernetes authenticates with Vault by passing the ServiceAccount + token stored in the named Secret resource to the Vault server. type: object required: - role properties: mountPath: - description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. + description: |- + The Vault mountPath here is the mount path to use when authenticating with + Vault. For example, setting a value to `/v1/auth/foo`, will use the path + `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the + default value "/v1/auth/kubernetes" will be used. type: string role: - description: A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies. + description: |- + A required field containing the Vault Role to assume. A Role binds a + Kubernetes ServiceAccount with a set of Vault policies. type: string secretRef: - description: The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported. + description: |- + The required Secret field containing a Kubernetes ServiceAccount JWT used + for authenticating with Vault. Use of 'ambient credentials' is not + supported. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceAccountRef: - description: A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. + description: |- + A reference to a service account that will be used to request a bound + token (also known as "projected token"). Compared to using "secretRef", + using this field means that you don't rely on statically bound tokens. To + use this field, you must configure an RBAC rule to let cert-manager + request a token. type: object required: - name properties: + audiences: + description: |- + TokenAudiences is an optional list of extra audiences to include in the token passed to Vault. The default token + consisting of the issuer's namespace and name is always included. + type: array + items: + type: string name: description: Name of the ServiceAccount used to request a token. type: string @@ -2822,44 +5224,112 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string caBundle: - description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by Vault. Only used if using HTTPS to connect to Vault and ignored for HTTP connections. Mutually exclusive with CABundleSecretRef. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. + description: |- + Base64-encoded bundle of PEM CAs which will be used to validate the certificate + chain presented by Vault. Only used if using HTTPS to connect to Vault and + ignored for HTTP connections. + Mutually exclusive with CABundleSecretRef. + If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in + the cert-manager controller container is used to validate the TLS connection. type: string format: byte caBundleSecretRef: - description: Reference to a Secret containing a bundle of PEM-encoded CAs to use when verifying the certificate chain presented by Vault when using HTTPS. Mutually exclusive with CABundle. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. If no key for the Secret is specified, cert-manager will default to 'ca.crt'. + description: |- + Reference to a Secret containing a bundle of PEM-encoded CAs to use when + verifying the certificate chain presented by Vault when using HTTPS. + Mutually exclusive with CABundle. + If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in + the cert-manager controller container is used to validate the TLS connection. + If no key for the Secret is specified, cert-manager will default to 'ca.crt'. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + clientCertSecretRef: + description: |- + Reference to a Secret containing a PEM-encoded Client Certificate to use when the + Vault server requires mTLS. + type: object + required: + - name + properties: + key: + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. + type: string + name: + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + clientKeySecretRef: + description: |- + Reference to a Secret containing a PEM-encoded Client Private Key to use when the + Vault server requires mTLS. + type: object + required: + - name + properties: + key: + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. + type: string + name: + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces' + description: |- + Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" + More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces type: string path: - description: 'Path is the mount path of the Vault PKI backend''s `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name".' + description: |- + Path is the mount path of the Vault PKI backend's `sign` endpoint, e.g: + "my_pki_mount/sign/my-role-name". type: string server: description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".' type: string venafi: - description: Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone. + description: |- + Venafi configures this issuer to sign certificates using a Venafi TPP + or Venafi Cloud policy zone. type: object required: - zone properties: cloud: - description: Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified. + description: |- + Cloud specifies the Venafi cloud configuration settings. + Only one of TPP or Cloud may be specified. type: object required: - apiTokenSecretRef @@ -2871,59 +5341,96 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: - description: URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1". + description: |- + URL is the base URL for Venafi Cloud. + Defaults to "https://api.venafi.cloud/v1". type: string tpp: - description: TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified. + description: |- + TPP specifies Trust Protection Platform configuration settings. + Only one of TPP or Cloud may be specified. type: object required: - credentialsRef - url properties: caBundle: - description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. If undefined, the certificate bundle in the cert-manager controller container is used to validate the chain. + description: |- + Base64-encoded bundle of PEM CAs which will be used to validate the certificate + chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. + If undefined, the certificate bundle in the cert-manager controller container + is used to validate the chain. type: string format: byte credentialsRef: - description: CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'. + description: |- + CredentialsRef is a reference to a Secret containing the username and + password for the TPP server. + The secret must contain two keys, 'username' and 'password'. type: object required: - name properties: name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: - description: 'URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk".' + description: |- + URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, + for example: "https://tpp.example.com/vedsdk". type: string zone: - description: Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required. + description: |- + Zone is the Venafi Policy Zone to use for this issuer. + All requests made to the Venafi platform will be restricted by the named + zone policy. + This field is required. type: string status: description: Status of the ClusterIssuer. This is set and managed automatically. type: object properties: acme: - description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. + description: |- + ACME specific status options. + This field should only be set if the Issuer is configured to use an ACME + server to issue certificates. type: object properties: lastPrivateKeyHash: - description: LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + description: |- + LastPrivateKeyHash is a hash of the private key associated with the latest + registered ACME account, in order to track changes made to registered account + associated with the Issuer type: string lastRegisteredEmail: - description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + description: |- + LastRegisteredEmail is the email associated with the latest registered + ACME account, in order to track changes made to registered account + associated with the Issuer type: string uri: - description: URI is the unique account identifier, which can also be used to retrieve account details from the CA + description: |- + URI is the unique account identifier, which can also be used to retrieve + account details from the CA type: string conditions: - description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`. + description: |- + List of status conditions to indicate the status of a CertificateRequest. + Known condition types are `Ready`. type: array items: description: IssuerCondition contains condition information for an Issuer. @@ -2933,18 +5440,29 @@ spec: - type properties: lastTransitionTime: - description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + description: |- + LastTransitionTime is the timestamp corresponding to the last status + change of this condition. type: string format: date-time message: - description: Message is a human readable description of the details of the last transition, complementing reason. + description: |- + Message is a human readable description of the details of the last + transition, complementing reason. type: string observedGeneration: - description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer. + description: |- + If set, this represents the .metadata.generation that the condition was + set based upon. + For instance, if .metadata.generation is currently 12, but the + .status.condition[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the Issuer. type: integer format: int64 reason: - description: Reason is a brief machine readable explanation for the condition's last transition. + description: |- + Reason is a brief machine readable explanation for the condition's last + transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). @@ -2961,15 +5479,24 @@ spec: x-kubernetes-list-type: map served: true storage: true + +# END crd {{- end }} + --- +# START crd {{- if or .Values.crds.enabled .Values.installCRDs }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: issuers.cert-manager.io + # START annotations {{- if .Values.crds.keep }} + annotations: + helm.sh/resource-policy: keep + # END annotations {{- end }} labels: app: '{{ template "cert-manager.name" . }}' app.kubernetes.io/name: '{{ template "cert-manager.name" . }}' - app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/component: "crds" # Generated labels {{- include "labels" . | nindent 4 }} spec: group: cert-manager.io @@ -2999,16 +5526,29 @@ spec: type: date schema: openAPIV3Schema: - description: An Issuer represents a certificate issuing authority which can be referenced as part of `issuerRef` fields. It is scoped to a single namespace and can therefore only be referenced by resources within the same namespace. + description: |- + An Issuer represents a certificate issuing authority which can be + referenced as part of `issuerRef` fields. + It is scoped to a single namespace and can therefore only be referenced by + resources within the same namespace. type: object required: - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -3017,34 +5557,65 @@ spec: type: object properties: acme: - description: ACME configures this issuer to communicate with a RFC8555 (ACME) server to obtain signed x509 certificates. + description: |- + ACME configures this issuer to communicate with a RFC8555 (ACME) server + to obtain signed x509 certificates. type: object required: - privateKeySecretRef - server properties: caBundle: - description: Base64-encoded bundle of PEM CAs which can be used to validate the certificate chain presented by the ACME server. Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various kinds of security vulnerabilities. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. + description: |- + Base64-encoded bundle of PEM CAs which can be used to validate the certificate + chain presented by the ACME server. + Mutually exclusive with SkipTLSVerify; prefer using CABundle to prevent various + kinds of security vulnerabilities. + If CABundle and SkipTLSVerify are unset, the system certificate bundle inside + the container is used to validate the TLS connection. type: string format: byte disableAccountKeyGeneration: - description: Enables or disables generating a new ACME account key. If true, the Issuer resource will *not* request a new account but will expect the account key to be supplied via an existing secret. If false, the cert-manager system will generate a new ACME account key for the Issuer. Defaults to false. + description: |- + Enables or disables generating a new ACME account key. + If true, the Issuer resource will *not* request a new account but will expect + the account key to be supplied via an existing secret. + If false, the cert-manager system will generate a new ACME account key + for the Issuer. + Defaults to false. type: boolean email: - description: Email is the email address to be associated with the ACME account. This field is optional, but it is strongly recommended to be set. It will be used to contact you in case of issues with your account or certificates, including expiry notification emails. This field may be updated after the account is initially registered. + description: |- + Email is the email address to be associated with the ACME account. + This field is optional, but it is strongly recommended to be set. + It will be used to contact you in case of issues with your account or + certificates, including expiry notification emails. + This field may be updated after the account is initially registered. type: string enableDurationFeature: - description: Enables requesting a Not After date on certificates that matches the duration of the certificate. This is not supported by all ACME servers like Let's Encrypt. If set to true when the ACME server does not support it it will create an error on the Order. Defaults to false. + description: |- + Enables requesting a Not After date on certificates that matches the + duration of the certificate. This is not supported by all ACME servers + like Let's Encrypt. If set to true when the ACME server does not support + it, it will create an error on the Order. + Defaults to false. type: boolean externalAccountBinding: - description: ExternalAccountBinding is a reference to a CA external account of the ACME server. If set, upon registration cert-manager will attempt to associate the given external account credentials with the registered ACME account. + description: |- + ExternalAccountBinding is a reference to a CA external account of the ACME + server. + If set, upon registration cert-manager will attempt to associate the given + external account credentials with the registered ACME account. type: object required: - keyID - keySecretRef properties: keyAlgorithm: - description: 'Deprecated: keyAlgorithm field exists for historical compatibility reasons and should not be used. The algorithm is now hardcoded to HS256 in golang/x/crypto/acme.' + description: |- + Deprecated: keyAlgorithm field exists for historical compatibility + reasons and should not be used. The algorithm is now hardcoded to HS256 + in golang/x/crypto/acme. type: string enum: - HS256 @@ -3054,68 +5625,130 @@ spec: description: keyID is the ID of the CA key that the External Account is bound to. type: string keySecretRef: - description: keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes Secret which holds the symmetric MAC key of the External Account Binding. The `key` is the index string that is paired with the key data in the Secret and should not be confused with the key data itself, or indeed with the External Account Binding keyID above. The secret key stored in the Secret **must** be un-padded, base64 URL encoded data. + description: |- + keySecretRef is a Secret Key Selector referencing a data item in a Kubernetes + Secret which holds the symmetric MAC key of the External Account Binding. + The `key` is the index string that is paired with the key data in the + Secret and should not be confused with the key data itself, or indeed with + the External Account Binding keyID above. + The secret key stored in the Secret **must** be un-padded, base64 URL + encoded data. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string preferredChain: - description: 'PreferredChain is the chain to use if the ACME server outputs multiple. PreferredChain is no guarantee that this one gets delivered by the ACME endpoint. For example, for Let''s Encrypt''s DST crosssign you would use: "DST Root CA X3" or "ISRG Root X1" for the newer Let''s Encrypt root CA. This value picks the first certificate bundle in the ACME alternative chains that has a certificate with this value as its issuer''s CN' + description: |- + PreferredChain is the chain to use if the ACME server outputs multiple. + PreferredChain is no guarantee that this one gets delivered by the ACME + endpoint. + For example, for Let's Encrypt's DST crosssign you would use: + "DST Root CA X3" or "ISRG Root X1" for the newer Let's Encrypt root CA. + This value picks the first certificate bundle in the combined set of + ACME default and alternative chains that has a root-most certificate with + this value as its issuer's commonname. type: string maxLength: 64 privateKeySecretRef: - description: PrivateKey is the name of a Kubernetes Secret resource that will be used to store the automatically generated ACME account private key. Optionally, a `key` may be specified to select a specific entry within the named Secret resource. If `key` is not specified, a default of `tls.key` will be used. + description: |- + PrivateKey is the name of a Kubernetes Secret resource that will be used to + store the automatically generated ACME account private key. + Optionally, a `key` may be specified to select a specific entry within + the named Secret resource. + If `key` is not specified, a default of `tls.key` will be used. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string server: - description: 'Server is the URL used to access the ACME server''s ''directory'' endpoint. For example, for Let''s Encrypt''s staging endpoint, you would use: "https://acme-staging-v02.api.letsencrypt.org/directory". Only ACME v2 endpoints (i.e. RFC 8555) are supported.' + description: |- + Server is the URL used to access the ACME server's 'directory' endpoint. + For example, for Let's Encrypt's staging endpoint, you would use: + "https://acme-staging-v02.api.letsencrypt.org/directory". + Only ACME v2 endpoints (i.e. RFC 8555) are supported. type: string skipTLSVerify: - description: 'INSECURE: Enables or disables validation of the ACME server TLS certificate. If true, requests to the ACME server will not have the TLS certificate chain validated. Mutually exclusive with CABundle; prefer using CABundle to prevent various kinds of security vulnerabilities. Only enable this option in development environments. If CABundle and SkipTLSVerify are unset, the system certificate bundle inside the container is used to validate the TLS connection. Defaults to false.' + description: |- + INSECURE: Enables or disables validation of the ACME server TLS certificate. + If true, requests to the ACME server will not have the TLS certificate chain + validated. + Mutually exclusive with CABundle; prefer using CABundle to prevent various + kinds of security vulnerabilities. + Only enable this option in development environments. + If CABundle and SkipTLSVerify are unset, the system certificate bundle inside + the container is used to validate the TLS connection. + Defaults to false. type: boolean solvers: - description: 'Solvers is a list of challenge solvers that will be used to solve ACME challenges for the matching domains. Solver configurations must be provided in order to obtain certificates from an ACME server. For more information, see: https://cert-manager.io/docs/configuration/acme/' + description: |- + Solvers is a list of challenge solvers that will be used to solve + ACME challenges for the matching domains. + Solver configurations must be provided in order to obtain certificates + from an ACME server. + For more information, see: https://cert-manager.io/docs/configuration/acme/ type: array items: - description: An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. A selector may be provided to use different solving strategies for different DNS names. Only one of HTTP01 or DNS01 must be provided. + description: |- + An ACMEChallengeSolver describes how to solve ACME challenges for the issuer it is part of. + A selector may be provided to use different solving strategies for different DNS names. + Only one of HTTP01 or DNS01 must be provided. type: object properties: dns01: - description: Configures cert-manager to attempt to complete authorizations by performing the DNS01 challenge flow. + description: |- + Configures cert-manager to attempt to complete authorizations by + performing the DNS01 challenge flow. type: object properties: acmeDNS: - description: Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage DNS01 challenge records. + description: |- + Use the 'ACME DNS' (https://github.com/joohoi/acme-dns) API to manage + DNS01 challenge records. type: object required: - accountSecretRef - host properties: accountSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string host: type: string @@ -3129,40 +5762,61 @@ spec: - serviceConsumerDomain properties: accessTokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientSecretSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string clientTokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceConsumerDomain: type: string @@ -3174,19 +5828,30 @@ spec: - subscriptionID properties: clientID: - description: if both this and ClientSecret are left unset MSI will be used + description: |- + Auth: Azure Service Principal: + The ClientID of the Azure Service Principal used to authenticate with Azure DNS. + If set, ClientSecret and TenantID must also be set. type: string clientSecretSecretRef: - description: if both this and ClientID are left unset MSI will be used + description: |- + Auth: Azure Service Principal: + A reference to a Secret containing the password associated with the Service Principal. + If set, ClientID and TenantID must also be set. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string environment: description: name of the Azure environment (default AzurePublicCloud) @@ -3200,14 +5865,19 @@ spec: description: name of the DNS zone that should be used type: string managedIdentity: - description: managed identity configuration, can not be used at the same time as clientID, clientSecretSecretRef or tenantID + description: |- + Auth: Azure Workload Identity or Azure Managed Service Identity: + Settings to enable Azure Workload Identity or Azure Managed Service Identity + If set, ClientID, ClientSecret and TenantID must not be set. type: object properties: clientID: description: client ID of the managed identity, can not be used at the same time as resourceID type: string resourceID: - description: resource ID of the managed identity, can not be used at the same time as clientID + description: |- + resource ID of the managed identity, can not be used at the same time as clientID + Cannot be used for Azure Managed Service Identity type: string resourceGroupName: description: resource group the DNS zone is located in @@ -3216,7 +5886,10 @@ spec: description: ID of the Azure subscription type: string tenantID: - description: when specifying ClientID and ClientSecret then this field is also needed + description: |- + Auth: Azure Service Principal: + The TenantID of the Azure Service Principal used to authenticate with Azure DNS. + If set, ClientID and ClientSecret must also be set. type: string cloudDNS: description: Use the Google Cloud DNS API to manage DNS01 challenge records. @@ -3225,37 +5898,55 @@ spec: - project properties: hostedZoneName: - description: HostedZoneName is an optional field that tells cert-manager in which Cloud DNS zone the challenge record has to be created. If left empty cert-manager will automatically choose a zone. + description: |- + HostedZoneName is an optional field that tells cert-manager in which + Cloud DNS zone the challenge record has to be created. + If left empty cert-manager will automatically choose a zone. type: string project: type: string serviceAccountSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string cloudflare: description: Use the Cloudflare API to manage DNS01 challenge records. type: object properties: apiKeySecretRef: - description: 'API key to use to authenticate with Cloudflare. Note: using an API token to authenticate is now the recommended method as it allows greater control of permissions.' + description: |- + API key to use to authenticate with Cloudflare. + Note: using an API token to authenticate is now the recommended method + as it allows greater control of permissions. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string apiTokenSecretRef: description: API token used to authenticate with Cloudflare. @@ -3264,16 +5955,23 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string email: description: Email of the account, only required when using API key based authentication. type: string cnameStrategy: - description: CNAMEStrategy configures how the DNS01 provider should handle CNAME records when found in DNS zones. + description: |- + CNAMEStrategy configures how the DNS01 provider should handle CNAME + records when found in DNS zones. type: string enum: - None @@ -3285,43 +5983,69 @@ spec: - tokenSecretRef properties: tokenSecretRef: - description: A reference to a specific 'key' within a Secret resource. In some instances, `key` is a required field. + description: |- + A reference to a specific 'key' within a Secret resource. + In some instances, `key` is a required field. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string rfc2136: - description: Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) to manage DNS01 challenge records. + description: |- + Use RFC2136 ("Dynamic Updates in the Domain Name System") (https://datatracker.ietf.org/doc/rfc2136/) + to manage DNS01 challenge records. type: object required: - nameserver properties: nameserver: - description: The IP address or hostname of an authoritative DNS server supporting RFC2136 in the form host:port. If the host is an IPv6 address it must be enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. This field is required. + description: |- + The IP address or hostname of an authoritative DNS server supporting + RFC2136 in the form host:port. If the host is an IPv6 address it must be + enclosed in square brackets (e.g [2001:db8::1]) ; port is optional. + This field is required. type: string tsigAlgorithm: - description: 'The TSIG Algorithm configured in the DNS supporting RFC2136. Used only when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. Supported values are (case-insensitive): ``HMACMD5`` (default), ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``.' + description: |- + The TSIG Algorithm configured in the DNS supporting RFC2136. Used only + when ``tsigSecretSecretRef`` and ``tsigKeyName`` are defined. + Supported values are (case-insensitive): ``HMACMD5`` (default), + ``HMACSHA1``, ``HMACSHA256`` or ``HMACSHA512``. type: string tsigKeyName: - description: The TSIG Key name configured in the DNS. If ``tsigSecretSecretRef`` is defined, this field is required. + description: |- + The TSIG Key name configured in the DNS. + If ``tsigSecretSecretRef`` is defined, this field is required. type: string tsigSecretSecretRef: - description: The name of the secret containing the TSIG value. If ``tsigKeyName`` is defined, this field is required. + description: |- + The name of the secret containing the TSIG value. + If ``tsigKeyName`` is defined, this field is required. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string route53: description: Use the AWS Route53 API to manage DNS01 challenge records. @@ -3330,20 +6054,71 @@ spec: - region properties: accessKeyID: - description: 'The AccessKeyID is used for authentication. Cannot be set when SecretAccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The AccessKeyID is used for authentication. + Cannot be set when SecretAccessKeyID is set. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: string accessKeyIDSecretRef: - description: 'The SecretAccessKey is used for authentication. If set, pull the AWS access key ID from a key within a Kubernetes Secret. Cannot be set when AccessKeyID is set. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The SecretAccessKey is used for authentication. If set, pull the AWS + access key ID from a key within a Kubernetes Secret. + Cannot be set when AccessKeyID is set. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string + auth: + description: Auth configures how cert-manager authenticates. + type: object + required: + - kubernetes + properties: + kubernetes: + description: |- + Kubernetes authenticates with Route53 using AssumeRoleWithWebIdentity + by passing a bound ServiceAccount token. + type: object + required: + - serviceAccountRef + properties: + serviceAccountRef: + description: |- + A reference to a service account that will be used to request a bound + token (also known as "projected token"). To use this field, you must + configure an RBAC rule to let cert-manager request a token. + type: object + required: + - name + properties: + audiences: + description: |- + TokenAudiences is an optional list of audiences to include in the + token passed to AWS. The default token consisting of the issuer's namespace + and name is always included. + If unset the audience defaults to `sts.amazonaws.com`. + type: array + items: + type: string + name: + description: Name of the ServiceAccount used to request a token. + type: string hostedZoneID: description: If set, the provider will manage only this zone in Route53 and will not do an lookup using the route53:ListHostedZonesByName api call. type: string @@ -3351,113 +6126,301 @@ spec: description: Always set the region when using AccessKeyID and SecretAccessKey type: string role: - description: Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata + description: |- + Role is a Role ARN which the Route53 provider will assume using either the explicit credentials AccessKeyID/SecretAccessKey + or the inferred credentials from environment variables, shared credentials file or AWS Instance metadata type: string secretAccessKeySecretRef: - description: 'The SecretAccessKey is used for authentication. If neither the Access Key nor Key ID are set, we fall-back to using env vars, shared credentials file or AWS Instance metadata, see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials' + description: |- + The SecretAccessKey is used for authentication. + If neither the Access Key nor Key ID are set, we fall-back to using env + vars, shared credentials file or AWS Instance metadata, + see: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html#specifying-credentials type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string webhook: - description: Configure an external webhook based DNS01 challenge solver to manage DNS01 challenge records. + description: |- + Configure an external webhook based DNS01 challenge solver to manage + DNS01 challenge records. type: object required: - groupName - solverName properties: config: - description: Additional configuration that should be passed to the webhook apiserver when challenges are processed. This can contain arbitrary JSON data. Secret values should not be specified in this stanza. If secret values are needed (e.g. credentials for a DNS service), you should use a SecretKeySelector to reference a Secret resource. For details on the schema of this field, consult the webhook provider implementation's documentation. + description: |- + Additional configuration that should be passed to the webhook apiserver + when challenges are processed. + This can contain arbitrary JSON data. + Secret values should not be specified in this stanza. + If secret values are needed (e.g. credentials for a DNS service), you + should use a SecretKeySelector to reference a Secret resource. + For details on the schema of this field, consult the webhook provider + implementation's documentation. x-kubernetes-preserve-unknown-fields: true groupName: - description: The API group name that should be used when POSTing ChallengePayload resources to the webhook apiserver. This should be the same as the GroupName specified in the webhook provider implementation. + description: |- + The API group name that should be used when POSTing ChallengePayload + resources to the webhook apiserver. + This should be the same as the GroupName specified in the webhook + provider implementation. type: string solverName: - description: The name of the solver to use, as defined in the webhook provider implementation. This will typically be the name of the provider, e.g. 'cloudflare'. + description: |- + The name of the solver to use, as defined in the webhook provider + implementation. + This will typically be the name of the provider, e.g. 'cloudflare'. type: string http01: - description: Configures cert-manager to attempt to complete authorizations by performing the HTTP01 challenge flow. It is not possible to obtain certificates for wildcard domain names (e.g. `*.example.com`) using the HTTP01 challenge mechanism. + description: |- + Configures cert-manager to attempt to complete authorizations by + performing the HTTP01 challenge flow. + It is not possible to obtain certificates for wildcard domain names + (e.g. `*.example.com`) using the HTTP01 challenge mechanism. type: object properties: gatewayHTTPRoute: - description: The Gateway API is a sig-network community API that models service networking in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will create HTTPRoutes with the specified labels in the same namespace as the challenge. This solver is experimental, and fields / behaviour may change in the future. + description: |- + The Gateway API is a sig-network community API that models service networking + in Kubernetes (https://gateway-api.sigs.k8s.io/). The Gateway solver will + create HTTPRoutes with the specified labels in the same namespace as the challenge. + This solver is experimental, and fields / behaviour may change in the future. type: object properties: labels: - description: Custom labels that will be applied to HTTPRoutes created by cert-manager while solving HTTP-01 challenges. + description: |- + Custom labels that will be applied to HTTPRoutes created by cert-manager + while solving HTTP-01 challenges. type: object additionalProperties: type: string parentRefs: - description: 'When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. cert-manager needs to know which parentRefs should be used when creating the HTTPRoute. Usually, the parentRef references a Gateway. See: https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways' + description: |- + When solving an HTTP-01 challenge, cert-manager creates an HTTPRoute. + cert-manager needs to know which parentRefs should be used when creating + the HTTPRoute. Usually, the parentRef references a Gateway. See: + https://gateway-api.sigs.k8s.io/api-types/httproute/#attaching-to-gateways type: array items: - description: "ParentReference identifies an API object (usually a Gateway) that can be considered a parent of this resource (usually a route). The only kind of parent resource with \"Core\" support is Gateway. This API may be extended in the future to support additional kinds of parent resources, such as HTTPRoute. \n The API object must be valid in the cluster; the Group and Kind must be registered in the cluster for this reference to be valid." + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + + This API may be extended in the future to support additional kinds of parent + resources. + + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. type: object required: - name properties: group: - description: "Group is the group of the referent. When unspecified, \"gateway.networking.k8s.io\" is inferred. To set the core API group (such as for a \"Service\" kind referent), Group must be explicitly set to \"\" (empty string). \n Support: Core" + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + + Support: Core type: string default: gateway.networking.k8s.io maxLength: 253 pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ kind: - description: "Kind is kind of the referent. \n Support: Core (Gateway) \n Support: Implementation-specific (Other Resources)" + description: |- + Kind is kind of the referent. + + + There are two kinds of parent resources with "Core" support: + + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + + Support for other resources is Implementation-Specific. type: string default: Gateway maxLength: 63 minLength: 1 pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ name: - description: "Name is the name of the referent. \n Support: Core" + description: |- + Name is the name of the referent. + + + Support: Core type: string maxLength: 253 minLength: 1 namespace: - description: "Namespace is the namespace of the referent. When unspecified, this refers to the local namespace of the Route. \n Note that there are specific rules for ParentRefs which cross namespace boundaries. Cross-namespace references are only valid if they are explicitly allowed by something in the namespace they are referring to. For example: Gateway has the AllowedRoutes field, and ReferenceGrant provides a generic way to enable any other kind of cross-namespace reference. \n Support: Core" + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + + Support: Core type: string maxLength: 63 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ port: - description: "Port is the network port this Route targets. It can be interpreted differently based on the type of parent resource. \n When the parent resource is a Gateway, this targets all listeners listening on the specified port that also support this kind of Route(and select this Route). It's not recommended to set `Port` unless the networking behaviors specified in a Route must apply to a specific port as opposed to a listener(s) whose port(s) may be changed. When both Port and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support other parent resources. Implementations supporting other types of parent resources MUST clearly document how/if Port is interpreted. \n For the purpose of status, an attachment is considered successful as long as the parent resource accepts it partially. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Extended \n " + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + + Support: Extended type: integer format: int32 maximum: 65535 minimum: 1 sectionName: - description: "SectionName is the name of a section within the target resource. In the following resources, SectionName is interpreted as the following: \n * Gateway: Listener Name. When both Port (experimental) and SectionName are specified, the name and port of the selected listener must match both specified values. \n Implementations MAY choose to support attaching Routes to other resources. If that is the case, they MUST clearly document how SectionName is interpreted. \n When unspecified (empty string), this will reference the entire resource. For the purpose of status, an attachment is considered successful if at least one section in the parent resource accepts it. For example, Gateway listeners can restrict which Routes can attach to them by Route kind, namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from the referencing Route, the Route MUST be considered successfully attached. If no Gateway listeners accept attachment from this Route, the Route MUST be considered detached from the Gateway. \n Support: Core" + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + + Support: Core type: string maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ serviceType: - description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + description: |- + Optional service type for Kubernetes solver service. Supported values + are NodePort or ClusterIP. If unset, defaults to NodePort. type: string ingress: - description: The ingress based HTTP01 challenge solver will solve challenges by creating or modifying Ingress resources in order to route requests for '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are provisioned by cert-manager for each Challenge to be completed. + description: |- + The ingress based HTTP01 challenge solver will solve challenges by + creating or modifying Ingress resources in order to route requests for + '/.well-known/acme-challenge/XYZ' to 'challenge solver' pods that are + provisioned by cert-manager for each Challenge to be completed. type: object properties: class: - description: This field configures the annotation `kubernetes.io/ingress.class` when creating Ingress resources to solve ACME challenges that use this challenge solver. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + This field configures the annotation `kubernetes.io/ingress.class` when + creating Ingress resources to solve ACME challenges that use this + challenge solver. Only one of `class`, `name` or `ingressClassName` may + be specified. type: string ingressClassName: - description: This field configures the field `ingressClassName` on the created Ingress resources used to solve ACME challenges that use this challenge solver. This is the recommended way of configuring the ingress class. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + This field configures the field `ingressClassName` on the created Ingress + resources used to solve ACME challenges that use this challenge solver. + This is the recommended way of configuring the ingress class. Only one of + `class`, `name` or `ingressClassName` may be specified. type: string ingressTemplate: - description: Optional ingress template used to configure the ACME challenge solver ingress used for HTTP01 challenges. + description: |- + Optional ingress template used to configure the ACME challenge solver + ingress used for HTTP01 challenges. type: object properties: metadata: - description: ObjectMeta overrides for the ingress used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + description: |- + ObjectMeta overrides for the ingress used to solve HTTP01 challenges. + Only the 'labels' and 'annotations' fields may be set. + If labels or annotations overlap with in-built values, the values here + will override the in-built values. type: object properties: annotations: @@ -3471,14 +6434,26 @@ spec: additionalProperties: type: string name: - description: The name of the ingress resource that should have ACME challenge solving routes inserted into it in order to solve HTTP01 challenges. This is typically used in conjunction with ingress controllers like ingress-gce, which maintains a 1:1 mapping between external IPs and ingress resources. Only one of `class`, `name` or `ingressClassName` may be specified. + description: |- + The name of the ingress resource that should have ACME challenge solving + routes inserted into it in order to solve HTTP01 challenges. + This is typically used in conjunction with ingress controllers like + ingress-gce, which maintains a 1:1 mapping between external IPs and + ingress resources. Only one of `class`, `name` or `ingressClassName` may + be specified. type: string podTemplate: - description: Optional pod template used to configure the ACME challenge solver pods used for HTTP01 challenges. + description: |- + Optional pod template used to configure the ACME challenge solver pods + used for HTTP01 challenges. type: object properties: metadata: - description: ObjectMeta overrides for the pod used to solve HTTP01 challenges. Only the 'labels' and 'annotations' fields may be set. If labels or annotations overlap with in-built values, the values here will override the in-built values. + description: |- + ObjectMeta overrides for the pod used to solve HTTP01 challenges. + Only the 'labels' and 'annotations' fields may be set. + If labels or annotations overlap with in-built values, the values here + will override the in-built values. type: object properties: annotations: @@ -3492,7 +6467,10 @@ spec: additionalProperties: type: string spec: - description: PodSpec defines overrides for the HTTP01 challenge solver pod. Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. All other fields will be ignored. + description: |- + PodSpec defines overrides for the HTTP01 challenge solver pod. + Check ACMEChallengeSolverHTTP01IngressPodSpec to find out currently supported fields. + All other fields will be ignored. type: object properties: affinity: @@ -3504,10 +6482,21 @@ spec: type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. type: array items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). type: object required: - preference @@ -3521,7 +6510,9 @@ spec: description: A list of node selector requirements by node's labels. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -3531,18 +6522,29 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -3552,20 +6554,35 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic weight: description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. type: object required: - nodeSelectorTerms @@ -3574,14 +6591,19 @@ spec: description: Required. A list of node selector terms. The terms are ORed. type: array items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. type: object properties: matchExpressions: description: A list of node selector requirements by node's labels. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -3591,18 +6613,29 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchFields: description: A list of node selector requirements by node's fields. type: array items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. type: object required: - key @@ -3612,21 +6645,40 @@ spec: description: The label key that the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic + x-kubernetes-list-type: atomic x-kubernetes-map-type: atomic podAffinity: description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -3642,14 +6694,18 @@ spec: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3659,28 +6715,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3690,49 +6794,90 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3742,28 +6887,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3773,33 +6966,64 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string + x-kubernetes-list-type: atomic podAntiAffinity: description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). type: object properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. type: array items: description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) @@ -3815,14 +7039,18 @@ spec: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3832,28 +7060,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3863,49 +7139,90 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. type: integer format: int32 + x-kubernetes-list-type: atomic requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. type: array items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running type: object required: - topologyKey properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3915,28 +7232,76 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + type: array + items: + type: string + x-kubernetes-list-type: atomic namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. type: object properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. type: array items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. type: object required: - key @@ -3946,40 +7311,75 @@ spec: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. type: array items: type: string + x-kubernetes-list-type: atomic + x-kubernetes-list-type: atomic matchLabels: - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object additionalProperties: type: string x-kubernetes-map-type: atomic namespaces: - description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace". + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". type: array items: type: string + x-kubernetes-list-type: atomic topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. type: string + x-kubernetes-list-type: atomic imagePullSecrets: description: If specified, the pod's imagePullSecrets type: array items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. type: object properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string + default: "" x-kubernetes-map-type: atomic nodeSelector: - description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ type: object additionalProperties: type: string @@ -3993,77 +7393,146 @@ spec: description: If specified, the pod's tolerations. type: array items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . type: object properties: effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. type: string key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. type: string operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. type: string tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. type: integer format: int64 value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. type: string serviceType: - description: Optional service type for Kubernetes solver service. Supported values are NodePort or ClusterIP. If unset, defaults to NodePort. + description: |- + Optional service type for Kubernetes solver service. Supported values + are NodePort or ClusterIP. If unset, defaults to NodePort. type: string selector: - description: Selector selects a set of DNSNames on the Certificate resource that should be solved using this challenge solver. If not specified, the solver will be treated as the 'default' solver with the lowest priority, i.e. if any other solver has a more specific match, it will be used instead. + description: |- + Selector selects a set of DNSNames on the Certificate resource that + should be solved using this challenge solver. + If not specified, the solver will be treated as the 'default' solver + with the lowest priority, i.e. if any other solver has a more specific + match, it will be used instead. type: object properties: dnsNames: - description: List of DNSNames that this solver will be used to solve. If specified and a match is found, a dnsNames selector will take precedence over a dnsZones selector. If multiple solvers match with the same dnsNames value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + description: |- + List of DNSNames that this solver will be used to solve. + If specified and a match is found, a dnsNames selector will take + precedence over a dnsZones selector. + If multiple solvers match with the same dnsNames value, the solver + with the most matching labels in matchLabels will be selected. + If neither has more matches, the solver defined earlier in the list + will be selected. type: array items: type: string dnsZones: - description: List of DNSZones that this solver will be used to solve. The most specific DNS zone match specified here will take precedence over other DNS zone matches, so a solver specifying sys.example.com will be selected over one specifying example.com for the domain www.sys.example.com. If multiple solvers match with the same dnsZones value, the solver with the most matching labels in matchLabels will be selected. If neither has more matches, the solver defined earlier in the list will be selected. + description: |- + List of DNSZones that this solver will be used to solve. + The most specific DNS zone match specified here will take precedence + over other DNS zone matches, so a solver specifying sys.example.com + will be selected over one specifying example.com for the domain + www.sys.example.com. + If multiple solvers match with the same dnsZones value, the solver + with the most matching labels in matchLabels will be selected. + If neither has more matches, the solver defined earlier in the list + will be selected. type: array items: type: string matchLabels: - description: A label selector that is used to refine the set of certificate's that this challenge solver will apply to. + description: |- + A label selector that is used to refine the set of certificate's that + this challenge solver will apply to. type: object additionalProperties: type: string ca: - description: CA configures this issuer to sign certificates using a signing CA keypair stored in a Secret resource. This is used to build internal PKIs that are managed by cert-manager. + description: |- + CA configures this issuer to sign certificates using a signing CA keypair + stored in a Secret resource. + This is used to build internal PKIs that are managed by cert-manager. type: object required: - secretName properties: crlDistributionPoints: - description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set, certificates will be issued without distribution points set. + description: |- + The CRL distribution points is an X.509 v3 certificate extension which identifies + the location of the CRL from which the revocation of this certificate can be checked. + If not set, certificates will be issued without distribution points set. + type: array + items: + type: string + issuingCertificateURLs: + description: |- + IssuingCertificateURLs is a list of URLs which this issuer should embed into certificates + it creates. See https://www.rfc-editor.org/rfc/rfc5280#section-4.2.2.1 for more details. + As an example, such a URL might be "http://ca.domain.com/ca.crt". type: array items: type: string ocspServers: - description: The OCSP server list is an X.509 v3 extension that defines a list of URLs of OCSP responders. The OCSP responders can be queried for the revocation status of an issued certificate. If not set, the certificate will be issued with no OCSP servers set. For example, an OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". + description: |- + The OCSP server list is an X.509 v3 extension that defines a list of + URLs of OCSP responders. The OCSP responders can be queried for the + revocation status of an issued certificate. If not set, the + certificate will be issued with no OCSP servers set. For example, an + OCSP server URL could be "http://ocsp.int-x3.letsencrypt.org". type: array items: type: string secretName: - description: SecretName is the name of the secret used to sign Certificates issued by this Issuer. + description: |- + SecretName is the name of the secret used to sign Certificates issued + by this Issuer. type: string selfSigned: - description: SelfSigned configures this issuer to 'self sign' certificates using the private key used to create the CertificateRequest object. + description: |- + SelfSigned configures this issuer to 'self sign' certificates using the + private key used to create the CertificateRequest object. type: object properties: crlDistributionPoints: - description: The CRL distribution points is an X.509 v3 certificate extension which identifies the location of the CRL from which the revocation of this certificate can be checked. If not set certificate will be issued without CDP. Values are strings. + description: |- + The CRL distribution points is an X.509 v3 certificate extension which identifies + the location of the CRL from which the revocation of this certificate can be checked. + If not set certificate will be issued without CDP. Values are strings. type: array items: type: string vault: - description: Vault configures this issuer to sign certificates using a HashiCorp Vault PKI backend. + description: |- + Vault configures this issuer to sign certificates using a HashiCorp Vault + PKI backend. type: object required: - auth @@ -4075,7 +7544,9 @@ spec: type: object properties: appRole: - description: AppRole authenticates with Vault using the App Role auth mechanism, with the role and secret stored in a Kubernetes Secret resource. + description: |- + AppRole authenticates with Vault using the App Role auth mechanism, + with the role and secret stored in a Kubernetes Secret resource. type: object required: - path @@ -4083,53 +7554,94 @@ spec: - secretRef properties: path: - description: 'Path where the App Role authentication backend is mounted in Vault, e.g: "approle"' + description: |- + Path where the App Role authentication backend is mounted in Vault, e.g: + "approle" type: string roleId: - description: RoleID configured in the App Role authentication backend when setting up the authentication backend in Vault. + description: |- + RoleID configured in the App Role authentication backend when setting + up the authentication backend in Vault. type: string secretRef: - description: Reference to a key in a Secret that contains the App Role secret used to authenticate with Vault. The `key` field must be specified and denotes which entry within the Secret resource is used as the app role secret. + description: |- + Reference to a key in a Secret that contains the App Role secret used + to authenticate with Vault. + The `key` field must be specified and denotes which entry within the Secret + resource is used as the app role secret. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string kubernetes: - description: Kubernetes authenticates with Vault by passing the ServiceAccount token stored in the named Secret resource to the Vault server. + description: |- + Kubernetes authenticates with Vault by passing the ServiceAccount + token stored in the named Secret resource to the Vault server. type: object required: - role properties: mountPath: - description: The Vault mountPath here is the mount path to use when authenticating with Vault. For example, setting a value to `/v1/auth/foo`, will use the path `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the default value "/v1/auth/kubernetes" will be used. + description: |- + The Vault mountPath here is the mount path to use when authenticating with + Vault. For example, setting a value to `/v1/auth/foo`, will use the path + `/v1/auth/foo/login` to authenticate with Vault. If unspecified, the + default value "/v1/auth/kubernetes" will be used. type: string role: - description: A required field containing the Vault Role to assume. A Role binds a Kubernetes ServiceAccount with a set of Vault policies. + description: |- + A required field containing the Vault Role to assume. A Role binds a + Kubernetes ServiceAccount with a set of Vault policies. type: string secretRef: - description: The required Secret field containing a Kubernetes ServiceAccount JWT used for authenticating with Vault. Use of 'ambient credentials' is not supported. + description: |- + The required Secret field containing a Kubernetes ServiceAccount JWT used + for authenticating with Vault. Use of 'ambient credentials' is not + supported. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string serviceAccountRef: - description: A reference to a service account that will be used to request a bound token (also known as "projected token"). Compared to using "secretRef", using this field means that you don't rely on statically bound tokens. To use this field, you must configure an RBAC rule to let cert-manager request a token. + description: |- + A reference to a service account that will be used to request a bound + token (also known as "projected token"). Compared to using "secretRef", + using this field means that you don't rely on statically bound tokens. To + use this field, you must configure an RBAC rule to let cert-manager + request a token. type: object required: - name properties: + audiences: + description: |- + TokenAudiences is an optional list of extra audiences to include in the token passed to Vault. The default token + consisting of the issuer's namespace and name is always included. + type: array + items: + type: string name: description: Name of the ServiceAccount used to request a token. type: string @@ -4140,44 +7652,112 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string caBundle: - description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by Vault. Only used if using HTTPS to connect to Vault and ignored for HTTP connections. Mutually exclusive with CABundleSecretRef. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. + description: |- + Base64-encoded bundle of PEM CAs which will be used to validate the certificate + chain presented by Vault. Only used if using HTTPS to connect to Vault and + ignored for HTTP connections. + Mutually exclusive with CABundleSecretRef. + If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in + the cert-manager controller container is used to validate the TLS connection. type: string format: byte caBundleSecretRef: - description: Reference to a Secret containing a bundle of PEM-encoded CAs to use when verifying the certificate chain presented by Vault when using HTTPS. Mutually exclusive with CABundle. If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in the cert-manager controller container is used to validate the TLS connection. If no key for the Secret is specified, cert-manager will default to 'ca.crt'. + description: |- + Reference to a Secret containing a bundle of PEM-encoded CAs to use when + verifying the certificate chain presented by Vault when using HTTPS. + Mutually exclusive with CABundle. + If neither CABundle nor CABundleSecretRef are defined, the certificate bundle in + the cert-manager controller container is used to validate the TLS connection. + If no key for the Secret is specified, cert-manager will default to 'ca.crt'. type: object required: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + clientCertSecretRef: + description: |- + Reference to a Secret containing a PEM-encoded Client Certificate to use when the + Vault server requires mTLS. + type: object + required: + - name + properties: + key: + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. + type: string + name: + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + clientKeySecretRef: + description: |- + Reference to a Secret containing a PEM-encoded Client Private Key to use when the + Vault server requires mTLS. + type: object + required: + - name + properties: + key: + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. + type: string + name: + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces' + description: |- + Name of the vault namespace. Namespaces is a set of features within Vault Enterprise that allows Vault environments to support Secure Multi-tenancy. e.g: "ns1" + More about namespaces can be found here https://www.vaultproject.io/docs/enterprise/namespaces type: string path: - description: 'Path is the mount path of the Vault PKI backend''s `sign` endpoint, e.g: "my_pki_mount/sign/my-role-name".' + description: |- + Path is the mount path of the Vault PKI backend's `sign` endpoint, e.g: + "my_pki_mount/sign/my-role-name". type: string server: description: 'Server is the connection address for the Vault server, e.g: "https://vault.example.com:8200".' type: string venafi: - description: Venafi configures this issuer to sign certificates using a Venafi TPP or Venafi Cloud policy zone. + description: |- + Venafi configures this issuer to sign certificates using a Venafi TPP + or Venafi Cloud policy zone. type: object required: - zone properties: cloud: - description: Cloud specifies the Venafi cloud configuration settings. Only one of TPP or Cloud may be specified. + description: |- + Cloud specifies the Venafi cloud configuration settings. + Only one of TPP or Cloud may be specified. type: object required: - apiTokenSecretRef @@ -4189,59 +7769,96 @@ spec: - name properties: key: - description: The key of the entry in the Secret resource's `data` field to be used. Some instances of this field may be defaulted, in others it may be required. + description: |- + The key of the entry in the Secret resource's `data` field to be used. + Some instances of this field may be defaulted, in others it may be + required. type: string name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: - description: URL is the base URL for Venafi Cloud. Defaults to "https://api.venafi.cloud/v1". + description: |- + URL is the base URL for Venafi Cloud. + Defaults to "https://api.venafi.cloud/v1". type: string tpp: - description: TPP specifies Trust Protection Platform configuration settings. Only one of TPP or Cloud may be specified. + description: |- + TPP specifies Trust Protection Platform configuration settings. + Only one of TPP or Cloud may be specified. type: object required: - credentialsRef - url properties: caBundle: - description: Base64-encoded bundle of PEM CAs which will be used to validate the certificate chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. If undefined, the certificate bundle in the cert-manager controller container is used to validate the chain. + description: |- + Base64-encoded bundle of PEM CAs which will be used to validate the certificate + chain presented by the TPP server. Only used if using HTTPS; ignored for HTTP. + If undefined, the certificate bundle in the cert-manager controller container + is used to validate the chain. type: string format: byte credentialsRef: - description: CredentialsRef is a reference to a Secret containing the username and password for the TPP server. The secret must contain two keys, 'username' and 'password'. + description: |- + CredentialsRef is a reference to a Secret containing the username and + password for the TPP server. + The secret must contain two keys, 'username' and 'password'. type: object required: - name properties: name: - description: 'Name of the resource being referred to. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the resource being referred to. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string url: - description: 'URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, for example: "https://tpp.example.com/vedsdk".' + description: |- + URL is the base URL for the vedsdk endpoint of the Venafi TPP instance, + for example: "https://tpp.example.com/vedsdk". type: string zone: - description: Zone is the Venafi Policy Zone to use for this issuer. All requests made to the Venafi platform will be restricted by the named zone policy. This field is required. + description: |- + Zone is the Venafi Policy Zone to use for this issuer. + All requests made to the Venafi platform will be restricted by the named + zone policy. + This field is required. type: string status: description: Status of the Issuer. This is set and managed automatically. type: object properties: acme: - description: ACME specific status options. This field should only be set if the Issuer is configured to use an ACME server to issue certificates. + description: |- + ACME specific status options. + This field should only be set if the Issuer is configured to use an ACME + server to issue certificates. type: object properties: lastPrivateKeyHash: - description: LastPrivateKeyHash is a hash of the private key associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + description: |- + LastPrivateKeyHash is a hash of the private key associated with the latest + registered ACME account, in order to track changes made to registered account + associated with the Issuer type: string lastRegisteredEmail: - description: LastRegisteredEmail is the email associated with the latest registered ACME account, in order to track changes made to registered account associated with the Issuer + description: |- + LastRegisteredEmail is the email associated with the latest registered + ACME account, in order to track changes made to registered account + associated with the Issuer type: string uri: - description: URI is the unique account identifier, which can also be used to retrieve account details from the CA + description: |- + URI is the unique account identifier, which can also be used to retrieve + account details from the CA type: string conditions: - description: List of status conditions to indicate the status of a CertificateRequest. Known condition types are `Ready`. + description: |- + List of status conditions to indicate the status of a CertificateRequest. + Known condition types are `Ready`. type: array items: description: IssuerCondition contains condition information for an Issuer. @@ -4251,18 +7868,29 @@ spec: - type properties: lastTransitionTime: - description: LastTransitionTime is the timestamp corresponding to the last status change of this condition. + description: |- + LastTransitionTime is the timestamp corresponding to the last status + change of this condition. type: string format: date-time message: - description: Message is a human readable description of the details of the last transition, complementing reason. + description: |- + Message is a human readable description of the details of the last + transition, complementing reason. type: string observedGeneration: - description: If set, this represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the Issuer. + description: |- + If set, this represents the .metadata.generation that the condition was + set based upon. + For instance, if .metadata.generation is currently 12, but the + .status.condition[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the Issuer. type: integer format: int64 reason: - description: Reason is a brief machine readable explanation for the condition's last transition. + description: |- + Reason is a brief machine readable explanation for the condition's last + transition. type: string status: description: Status of the condition, one of (`True`, `False`, `Unknown`). @@ -4279,15 +7907,24 @@ spec: x-kubernetes-list-type: map served: true storage: true + +# END crd {{- end }} + --- +# START crd {{- if or .Values.crds.enabled .Values.installCRDs }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: orders.acme.cert-manager.io + # START annotations {{- if .Values.crds.keep }} + annotations: + helm.sh/resource-policy: keep + # END annotations {{- end }} labels: app: '{{ template "cert-manager.name" . }}' app.kubernetes.io/name: '{{ template "cert-manager.name" . }}' app.kubernetes.io/instance: '{{ .Release.Name }}' + app.kubernetes.io/component: "crds" # Generated labels {{- include "labels" . | nindent 4 }} spec: group: acme.cert-manager.io @@ -4329,10 +7966,19 @@ spec: - spec properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -4343,23 +7989,39 @@ spec: - request properties: commonName: - description: CommonName is the common name as specified on the DER encoded CSR. If specified, this value must also be present in `dnsNames` or `ipAddresses`. This field must match the corresponding field on the DER encoded CSR. + description: |- + CommonName is the common name as specified on the DER encoded CSR. + If specified, this value must also be present in `dnsNames` or `ipAddresses`. + This field must match the corresponding field on the DER encoded CSR. type: string dnsNames: - description: DNSNames is a list of DNS names that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR. + description: |- + DNSNames is a list of DNS names that should be included as part of the Order + validation process. + This field must match the corresponding field on the DER encoded CSR. type: array items: type: string duration: - description: Duration is the duration for the not after date for the requested certificate. this is set on order creation as pe the ACME spec. + description: |- + Duration is the duration for the not after date for the requested certificate. + this is set on order creation as pe the ACME spec. type: string ipAddresses: - description: IPAddresses is a list of IP addresses that should be included as part of the Order validation process. This field must match the corresponding field on the DER encoded CSR. + description: |- + IPAddresses is a list of IP addresses that should be included as part of the Order + validation process. + This field must match the corresponding field on the DER encoded CSR. type: array items: type: string issuerRef: - description: IssuerRef references a properly configured ACME-type Issuer which should be used to create this Order. If the Issuer does not exist, processing will be retried. If the Issuer is not an 'ACME' Issuer, an error will be returned and the Order will be marked as failed. + description: |- + IssuerRef references a properly configured ACME-type Issuer which should + be used to create this Order. + If the Issuer does not exist, processing will be retried. + If the Issuer is not an 'ACME' Issuer, an error will be returned and the + Order will be marked as failed. type: object required: - name @@ -4374,26 +8036,42 @@ spec: description: Name of the resource being referred to. type: string request: - description: Certificate signing request bytes in DER encoding. This will be used when finalizing the order. This field must be set on the order. + description: |- + Certificate signing request bytes in DER encoding. + This will be used when finalizing the order. + This field must be set on the order. type: string format: byte status: type: object properties: authorizations: - description: Authorizations contains data returned from the ACME server on what authorizations must be completed in order to validate the DNS names specified on the Order. + description: |- + Authorizations contains data returned from the ACME server on what + authorizations must be completed in order to validate the DNS names + specified on the Order. type: array items: - description: ACMEAuthorization contains data returned from the ACME server on an authorization that must be completed in order validate a DNS name on an ACME Order resource. + description: |- + ACMEAuthorization contains data returned from the ACME server on an + authorization that must be completed in order validate a DNS name on an ACME + Order resource. type: object required: - url properties: challenges: - description: Challenges specifies the challenge types offered by the ACME server. One of these challenge types will be selected when validating the DNS name and an appropriate Challenge resource will be created to perform the ACME challenge process. + description: |- + Challenges specifies the challenge types offered by the ACME server. + One of these challenge types will be selected when validating the DNS + name and an appropriate Challenge resource will be created to perform + the ACME challenge process. type: array items: - description: Challenge specifies a challenge offered by the ACME server for an Order. An appropriate Challenge resource can be created to perform the ACME challenge process. + description: |- + Challenge specifies a challenge offered by the ACME server for an Order. + An appropriate Challenge resource can be created to perform the ACME + challenge process. type: object required: - token @@ -4401,19 +8079,36 @@ spec: - url properties: token: - description: Token is the token that must be presented for this challenge. This is used to compute the 'key' that must also be presented. + description: |- + Token is the token that must be presented for this challenge. + This is used to compute the 'key' that must also be presented. type: string type: - description: Type is the type of challenge being offered, e.g. 'http-01', 'dns-01', 'tls-sni-01', etc. This is the raw value retrieved from the ACME server. Only 'http-01' and 'dns-01' are supported by cert-manager, other values will be ignored. + description: |- + Type is the type of challenge being offered, e.g. 'http-01', 'dns-01', + 'tls-sni-01', etc. + This is the raw value retrieved from the ACME server. + Only 'http-01' and 'dns-01' are supported by cert-manager, other values + will be ignored. type: string url: - description: URL is the URL of this challenge. It can be used to retrieve additional metadata about the Challenge from the ACME server. + description: |- + URL is the URL of this challenge. It can be used to retrieve additional + metadata about the Challenge from the ACME server. type: string identifier: description: Identifier is the DNS name to be validated as part of this authorization type: string initialState: - description: InitialState is the initial state of the ACME authorization when first fetched from the ACME server. If an Authorization is already 'valid', the Order controller will not create a Challenge resource for the authorization. This will occur when working with an ACME server that enables 'authz reuse' (such as Let's Encrypt's production endpoint). If not set and 'identifier' is set, the state is assumed to be pending and a Challenge will be created. + description: |- + InitialState is the initial state of the ACME authorization when first + fetched from the ACME server. + If an Authorization is already 'valid', the Order controller will not + create a Challenge resource for the authorization. This will occur when + working with an ACME server that enables 'authz reuse' (such as Let's + Encrypt's production endpoint). + If not set and 'identifier' is set, the state is assumed to be pending + and a Challenge will be created. type: string enum: - valid @@ -4427,24 +8122,41 @@ spec: description: URL is the URL of the Authorization that must be completed type: string wildcard: - description: Wildcard will be true if this authorization is for a wildcard DNS name. If this is true, the identifier will be the *non-wildcard* version of the DNS name. For example, if '*.example.com' is the DNS name being validated, this field will be 'true' and the 'identifier' field will be 'example.com'. + description: |- + Wildcard will be true if this authorization is for a wildcard DNS name. + If this is true, the identifier will be the *non-wildcard* version of + the DNS name. + For example, if '*.example.com' is the DNS name being validated, this + field will be 'true' and the 'identifier' field will be 'example.com'. type: boolean certificate: - description: Certificate is a copy of the PEM encoded certificate for this Order. This field will be populated after the order has been successfully finalized with the ACME server, and the order has transitioned to the 'valid' state. + description: |- + Certificate is a copy of the PEM encoded certificate for this Order. + This field will be populated after the order has been successfully + finalized with the ACME server, and the order has transitioned to the + 'valid' state. type: string format: byte failureTime: - description: FailureTime stores the time that this order failed. This is used to influence garbage collection and back-off. + description: |- + FailureTime stores the time that this order failed. + This is used to influence garbage collection and back-off. type: string format: date-time finalizeURL: - description: FinalizeURL of the Order. This is used to obtain certificates for this order once it has been completed. + description: |- + FinalizeURL of the Order. + This is used to obtain certificates for this order once it has been completed. type: string reason: - description: Reason optionally provides more information about a why the order is in the current state. + description: |- + Reason optionally provides more information about a why the order is in + the current state. type: string state: - description: State contains the current state of this Order resource. States 'success' and 'expired' are 'final' + description: |- + State contains the current state of this Order resource. + States 'success' and 'expired' are 'final' type: string enum: - valid @@ -4455,8 +8167,13 @@ spec: - expired - errored url: - description: URL of the Order. This will initially be empty when the resource is first created. The Order controller will populate this field when the Order is first processed. This field will be immutable after it is initially set. + description: |- + URL of the Order. + This will initially be empty when the resource is first created. + The Order controller will populate this field when the Order is first processed. + This field will be immutable after it is initially set. type: string served: true storage: true -{{- end }} + +# END crd {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/deployment.yaml b/internal/constellation/helm/charts/cert-manager/templates/deployment.yaml index aea5736c0..e6f3f681e 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/deployment.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/deployment.yaml @@ -15,6 +15,10 @@ metadata: {{- end }} spec: replicas: {{ .Values.replicaCount }} + {{- /* The if statement below is equivalent to {{- if $value }} but will also return true for 0. */ -}} + {{- if not (has (quote .Values.global.revisionHistoryLimit) (list "" (quote ""))) }} + revisionHistoryLimit: {{ .Values.global.revisionHistoryLimit }} + {{- end }} selector: matchLabels: app.kubernetes.io/name: {{ template "cert-manager.name" . }} @@ -39,7 +43,7 @@ spec: annotations: {{- toYaml . | nindent 8 }} {{- end }} - {{- if and .Values.prometheus.enabled (not .Values.prometheus.servicemonitor.enabled) }} + {{- if and .Values.prometheus.enabled (not (or .Values.prometheus.servicemonitor.enabled .Values.prometheus.podmonitor.enabled)) }} {{- if not .Values.podAnnotations }} annotations: {{- end }} @@ -52,6 +56,7 @@ spec: {{- if hasKey .Values "automountServiceAccountToken" }} automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} {{- end }} + enableServiceLinks: {{ .Values.enableServiceLinks }} {{- with .Values.global.priorityClassName }} priorityClassName: {{ . | quote }} {{- end }} @@ -59,20 +64,30 @@ spec: securityContext: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- if or .Values.volumes .Values.config}} volumes: + {{- if .Values.config }} + - name: config + configMap: + name: {{ include "cert-manager.fullname" . }} + {{- end }} + {{ with .Values.volumes }} {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} containers: - name: {{ .Chart.Name }}-controller - {{- with .Values.image }} - image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}" - {{- end }} + image: "{{ template "image" (tuple .Values.image $.Chart.AppVersion) }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: - {{- if .Values.global.logLevel }} + {{- /* The if statement below is equivalent to {{- if $value }} but will also return true for 0. */ -}} + {{- if not (has (quote .Values.global.logLevel) (list "" (quote ""))) }} - --v={{ .Values.global.logLevel }} {{- end }} + {{- if .Values.config }} + - --config=/var/cert-manager/config/config.yaml + {{- end }} + {{- $config := default .Values.config "" }} {{- if .Values.clusterResourceNamespace }} - --cluster-resource-namespace={{ .Values.clusterResourceNamespace }} {{- else }} @@ -122,6 +137,9 @@ spec: {{- with .Values.dns01RecursiveNameservers }} - --dns01-recursive-nameservers={{ . }} {{- end }} + {{- if .Values.disableAutoApproval }} + - --controllers=-certificaterequests-approver + {{- end }} ports: - containerPort: 9402 name: http-metrics @@ -133,9 +151,15 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.volumeMounts }} + {{- if or .Values.config .Values.volumeMounts }} volumeMounts: + {{- if .Values.config }} + - name: config + mountPath: /var/cert-manager/config + {{- end }} + {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} env: - name: POD_NAMESPACE @@ -202,3 +226,6 @@ spec: dnsConfig: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.hostAliases }} + hostAliases: {{ toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/internal/constellation/helm/charts/cert-manager/templates/extras-objects.yaml b/internal/constellation/helm/charts/cert-manager/templates/extras-objects.yaml new file mode 100644 index 000000000..9ec3a7e9b --- /dev/null +++ b/internal/constellation/helm/charts/cert-manager/templates/extras-objects.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl . $ }} +{{ end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-egress.yaml b/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-egress.yaml index 09712009d..37f90bd2e 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-egress.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-egress.yaml @@ -11,13 +11,9 @@ spec: {{- end }} podSelector: matchLabels: - app: {{ include "webhook.name" . }} app.kubernetes.io/name: {{ include "webhook.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: "webhook" - {{- with .Values.webhook.podLabels }} - {{- toYaml . | nindent 6 }} - {{- end }} policyTypes: - Egress {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-webhooks.yaml b/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-webhooks.yaml index 349877a8b..3a0ed7a70 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-webhooks.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/networkpolicy-webhooks.yaml @@ -12,13 +12,9 @@ spec: {{- end }} podSelector: matchLabels: - app: {{ include "webhook.name" . }} - app.kubernetes.io/name: {{ include "webhook.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: "webhook" - {{- with .Values.webhook.podLabels }} - {{- toYaml . | nindent 6 }} - {{- end }} + app.kubernetes.io/name: {{ include "webhook.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: "webhook" policyTypes: - Ingress diff --git a/internal/constellation/helm/charts/cert-manager/templates/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cert-manager/templates/poddisruptionbudget.yaml index dab75ce68..ae71eed29 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/poddisruptionbudget.yaml @@ -17,10 +17,13 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: "controller" - {{- with .Values.podDisruptionBudget.minAvailable }} - minAvailable: {{ . }} + {{- if not (or (hasKey .Values.podDisruptionBudget "minAvailable") (hasKey .Values.podDisruptionBudget "maxUnavailable")) }} + minAvailable: 1 # Default value because minAvailable and maxUnavailable are not set {{- end }} - {{- with .Values.podDisruptionBudget.maxUnavailable }} - maxUnavailable: {{ . }} + {{- if hasKey .Values.podDisruptionBudget "minAvailable" }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if hasKey .Values.podDisruptionBudget "maxUnavailable" }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/podmonitor.yaml b/internal/constellation/helm/charts/cert-manager/templates/podmonitor.yaml new file mode 100644 index 000000000..1adc0609c --- /dev/null +++ b/internal/constellation/helm/charts/cert-manager/templates/podmonitor.yaml @@ -0,0 +1,50 @@ +{{- if and .Values.prometheus.enabled (and .Values.prometheus.podmonitor.enabled .Values.prometheus.servicemonitor.enabled) }} +{{- fail "Either .Values.prometheus.podmonitor.enabled or .Values.prometheus.servicemonitor.enabled can be enabled at a time, but not both." }} +{{- else if and .Values.prometheus.enabled .Values.prometheus.podmonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ template "cert-manager.fullname" . }} +{{- if .Values.prometheus.podmonitor.namespace }} + namespace: {{ .Values.prometheus.podmonitor.namespace }} +{{- else }} + namespace: {{ include "cert-manager.namespace" . }} +{{- end }} + labels: + app: {{ include "cert-manager.name" . }} + app.kubernetes.io/name: {{ include "cert-manager.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: "controller" + {{- include "labels" . | nindent 4 }} + prometheus: {{ .Values.prometheus.podmonitor.prometheusInstance }} + {{- with .Values.prometheus.podmonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.prometheus.podmonitor.annotations }} + annotations: + {{- with .Values.prometheus.podmonitor.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +spec: + jobLabel: {{ template "cert-manager.fullname" . }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "cert-manager.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: "controller" +{{- if .Values.prometheus.podmonitor.namespace }} + namespaceSelector: + matchNames: + - {{ include "cert-manager.namespace" . }} +{{- end }} + podMetricsEndpoints: + - port: http-metrics + path: {{ .Values.prometheus.podmonitor.path }} + interval: {{ .Values.prometheus.podmonitor.interval }} + scrapeTimeout: {{ .Values.prometheus.podmonitor.scrapeTimeout }} + honorLabels: {{ .Values.prometheus.podmonitor.honorLabels }} + {{- with .Values.prometheus.servicemonitor.endpointAdditionalProperties }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/rbac.yaml b/internal/constellation/helm/charts/cert-manager/templates/rbac.yaml index 830e37285..7a27d4f7a 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/rbac.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/rbac.yaml @@ -398,6 +398,26 @@ subjects: namespace: {{ include "cert-manager.namespace" . }} kind: ServiceAccount +{{- if .Values.global.rbac.aggregateClusterRoles }} +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "cert-manager.fullname" . }}-cluster-view + labels: + app: {{ include "cert-manager.name" . }} + app.kubernetes.io/name: {{ include "cert-manager.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: "controller" + {{- include "labels" . | nindent 4 }} + rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" +rules: + - apiGroups: ["cert-manager.io"] + resources: ["clusterissuers"] + verbs: ["get", "list", "watch"] + +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -414,6 +434,7 @@ metadata: rbac.authorization.k8s.io/aggregate-to-view: "true" rbac.authorization.k8s.io/aggregate-to-edit: "true" rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-cluster-reader: "true" {{- end }} rules: - apiGroups: ["cert-manager.io"] @@ -453,6 +474,8 @@ rules: --- +{{- if not .Values.disableAutoApproval -}} + # Permission to approve CertificateRequests referencing cert-manager.io Issuers and ClusterIssuers apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -468,7 +491,12 @@ rules: - apiGroups: ["cert-manager.io"] resources: ["signers"] verbs: ["approve"] - resourceNames: ["issuers.cert-manager.io/*", "clusterissuers.cert-manager.io/*"] + {{- with .Values.approveSignerNames }} + resourceNames: + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} --- @@ -493,6 +521,8 @@ subjects: --- +{{- end -}} + # Permission to: # - Update and sign CertificatSigningeRequests referencing cert-manager.io Issuers and ClusterIssuers # - Perform SubjectAccessReviews to test whether users are able to reference Namespaced Issuers diff --git a/internal/constellation/helm/charts/cert-manager/templates/service.yaml b/internal/constellation/helm/charts/cert-manager/templates/service.yaml index ec34d5878..360ec645e 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/service.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.prometheus.enabled }} +{{- if and .Values.prometheus.enabled (not .Values.prometheus.podmonitor.enabled) }} apiVersion: v1 kind: Service metadata: @@ -19,6 +19,12 @@ metadata: {{- end }} spec: type: ClusterIP + {{- if .Values.serviceIPFamilyPolicy }} + ipFamilyPolicy: {{ .Values.serviceIPFamilyPolicy }} + {{- end }} + {{- if .Values.serviceIPFamilies }} + ipFamilies: {{ .Values.serviceIPFamilies | toYaml | nindent 2 }} + {{- end }} ports: - protocol: TCP port: 9402 diff --git a/internal/constellation/helm/charts/cert-manager/templates/serviceaccount.yaml b/internal/constellation/helm/charts/cert-manager/templates/serviceaccount.yaml index 6026842ff..87fc00ea7 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/serviceaccount.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/serviceaccount.yaml @@ -20,6 +20,6 @@ metadata: app.kubernetes.io/component: "controller" {{- include "labels" . | nindent 4 }} {{- with .Values.serviceAccount.labels }} - {{ toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/servicemonitor.yaml b/internal/constellation/helm/charts/cert-manager/templates/servicemonitor.yaml index 9d9e89992..b63886077 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/servicemonitor.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/servicemonitor.yaml @@ -1,4 +1,6 @@ -{{- if and .Values.prometheus.enabled .Values.prometheus.servicemonitor.enabled }} +{{- if and .Values.prometheus.enabled (and .Values.prometheus.podmonitor.enabled .Values.prometheus.servicemonitor.enabled) }} +{{- fail "Either .Values.prometheus.podmonitor.enabled or .Values.prometheus.servicemonitor.enabled can be enabled at a time, but not both." }} +{{- else if and .Values.prometheus.enabled .Values.prometheus.servicemonitor.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -42,4 +44,7 @@ spec: interval: {{ .Values.prometheus.servicemonitor.interval }} scrapeTimeout: {{ .Values.prometheus.servicemonitor.scrapeTimeout }} honorLabels: {{ .Values.prometheus.servicemonitor.honorLabels }} + {{- with .Values.prometheus.servicemonitor.endpointAdditionalProperties }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/startupapicheck-job.yaml b/internal/constellation/helm/charts/cert-manager/templates/startupapicheck-job.yaml index a9b965e18..311b4c48e 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/startupapicheck-job.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/startupapicheck-job.yaml @@ -37,6 +37,7 @@ spec: {{- if hasKey .Values.startupapicheck "automountServiceAccountToken" }} automountServiceAccountToken: {{ .Values.startupapicheck.automountServiceAccountToken }} {{- end }} + enableServiceLinks: {{ .Values.startupapicheck.enableServiceLinks }} {{- with .Values.global.priorityClassName }} priorityClassName: {{ . | quote }} {{- end }} @@ -46,9 +47,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }}-startupapicheck - {{- with .Values.startupapicheck.image }} - image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}" - {{- end }} + image: "{{ template "image" (tuple .Values.startupapicheck.image $.Chart.AppVersion) }}" imagePullPolicy: {{ .Values.startupapicheck.image.pullPolicy }} args: - check diff --git a/internal/constellation/helm/charts/cert-manager/templates/webhook-config.yaml b/internal/constellation/helm/charts/cert-manager/templates/webhook-config.yaml index f3f72f02e..8f3ce20c3 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/webhook-config.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/webhook-config.yaml @@ -1,12 +1,6 @@ {{- if .Values.webhook.config -}} - {{- if not .Values.webhook.config.apiVersion -}} - {{- fail "webhook.config.apiVersion must be set" -}} - {{- end -}} - - {{- if not .Values.webhook.config.kind -}} - {{- fail "webhook.config.kind must be set" -}} - {{- end -}} -{{- end -}} +{{- $_ := .Values.webhook.config.apiVersion | required ".Values.webhook.config.apiVersion must be set !" -}} +{{- $_ := .Values.webhook.config.kind | required ".Values.webhook.config.kind must be set !" -}} apiVersion: v1 kind: ConfigMap metadata: @@ -19,7 +13,6 @@ metadata: app.kubernetes.io/component: "webhook" {{- include "labels" . | nindent 4 }} data: - {{- if .Values.webhook.config }} config.yaml: | - {{ .Values.webhook.config | toYaml | nindent 4 }} - {{- end }} + {{- .Values.webhook.config | toYaml | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/internal/constellation/helm/charts/cert-manager/templates/webhook-deployment.yaml b/internal/constellation/helm/charts/cert-manager/templates/webhook-deployment.yaml index 043c4b150..ae5399e90 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/webhook-deployment.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/webhook-deployment.yaml @@ -15,6 +15,10 @@ metadata: {{- end }} spec: replicas: {{ .Values.webhook.replicaCount }} + {{- /* The if statement below is equivalent to {{- if $value }} but will also return true for 0. */ -}} + {{- if not (has (quote .Values.global.revisionHistoryLimit) (list "" (quote ""))) }} + revisionHistoryLimit: {{ .Values.global.revisionHistoryLimit }} + {{- end }} selector: matchLabels: app.kubernetes.io/name: {{ include "webhook.name" . }} @@ -44,6 +48,7 @@ spec: {{- if hasKey .Values.webhook "automountServiceAccountToken" }} automountServiceAccountToken: {{ .Values.webhook.automountServiceAccountToken }} {{- end }} + enableServiceLinks: {{ .Values.webhook.enableServiceLinks }} {{- with .Values.global.priorityClassName }} priorityClassName: {{ . | quote }} {{- end }} @@ -54,14 +59,16 @@ spec: {{- if .Values.webhook.hostNetwork }} hostNetwork: true {{- end }} + {{- if .Values.webhook.hostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + {{- end }} containers: - name: {{ .Chart.Name }}-webhook - {{- with .Values.webhook.image }} - image: "{{- if .registry -}}{{ .registry }}/{{- end -}}{{ .repository }}{{- if (.digest) -}} @{{ .digest }}{{- else -}}:{{ default $.Chart.AppVersion .tag }} {{- end -}}" - {{- end }} + image: "{{ template "image" (tuple .Values.webhook.image $.Chart.AppVersion) }}" imagePullPolicy: {{ .Values.webhook.image.pullPolicy }} args: - {{- if .Values.global.logLevel }} + {{- /* The if statement below is equivalent to {{- if $value }} but will also return true for 0. */ -}} + {{- if not (has (quote .Values.global.logLevel) (list "" (quote ""))) }} - --v={{ .Values.global.logLevel }} {{- end }} {{- if .Values.webhook.config }} @@ -71,8 +78,8 @@ spec: {{ if not $config.securePort -}} - --secure-port={{ .Values.webhook.securePort }} {{- end }} - {{- if .Values.featureGates }} - - --feature-gates={{ .Values.featureGates }} + {{- if .Values.webhook.featureGates }} + - --feature-gates={{ .Values.webhook.featureGates }} {{- end }} {{- $tlsConfig := default $config.tlsConfig "" }} {{ if or (not $config.tlsConfig) (and (not $tlsConfig.dynamic) (not $tlsConfig.filesystem) ) -}} @@ -152,8 +159,8 @@ spec: - name: config mountPath: /var/cert-manager/config {{- end }} - {{- if .Values.webhook.volumeMounts }} - {{- toYaml .Values.webhook.volumeMounts | nindent 12 }} + {{- with .Values.webhook.volumeMounts }} + {{- toYaml . | nindent 12 }} {{- end }} {{- end }} {{- with .Values.webhook.nodeSelector }} @@ -179,7 +186,7 @@ spec: configMap: name: {{ include "webhook.fullname" . }} {{- end }} - {{- if .Values.webhook.volumes }} - {{- toYaml .Values.webhook.volumes | nindent 8 }} + {{- with .Values.webhook.volumes }} + {{- toYaml . | nindent 8 }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/webhook-mutating-webhook.yaml b/internal/constellation/helm/charts/cert-manager/templates/webhook-mutating-webhook.yaml index f3db011ef..9ea29777d 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/webhook-mutating-webhook.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/webhook-mutating-webhook.yaml @@ -15,17 +15,19 @@ metadata: {{- end }} webhooks: - name: webhook.cert-manager.io + {{- with .Values.webhook.mutatingWebhookConfiguration.namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 6 }} + {{- end }} rules: - apiGroups: - "cert-manager.io" - - "acme.cert-manager.io" apiVersions: - "v1" operations: - CREATE - - UPDATE resources: - - "*/*" + - "certificaterequests" admissionReviewVersions: ["v1"] # This webhook only accepts v1 cert-manager resources. # Equivalent matchPolicy ensures that non-v1 resource requests are sent to @@ -43,4 +45,4 @@ webhooks: name: {{ template "webhook.fullname" . }} namespace: {{ include "cert-manager.namespace" . }} path: /mutate - {{- end }} + {{- end }} \ No newline at end of file diff --git a/internal/constellation/helm/charts/cert-manager/templates/webhook-poddisruptionbudget.yaml b/internal/constellation/helm/charts/cert-manager/templates/webhook-poddisruptionbudget.yaml index c8a357cb1..ab2a48109 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/webhook-poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/webhook-poddisruptionbudget.yaml @@ -17,10 +17,13 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: "webhook" - {{- with .Values.webhook.podDisruptionBudget.minAvailable }} - minAvailable: {{ . }} + {{- if not (or (hasKey .Values.webhook.podDisruptionBudget "minAvailable") (hasKey .Values.webhook.podDisruptionBudget "maxUnavailable")) }} + minAvailable: 1 # Default value because minAvailable and maxUnavailable are not set {{- end }} - {{- with .Values.webhook.podDisruptionBudget.maxUnavailable }} - maxUnavailable: {{ . }} + {{- if hasKey .Values.webhook.podDisruptionBudget "minAvailable" }} + minAvailable: {{ .Values.webhook.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if hasKey .Values.webhook.podDisruptionBudget "maxUnavailable" }} + maxUnavailable: {{ .Values.webhook.podDisruptionBudget.maxUnavailable }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/webhook-service.yaml b/internal/constellation/helm/charts/cert-manager/templates/webhook-service.yaml index 5f9395049..86d47f164 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/webhook-service.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/webhook-service.yaml @@ -18,6 +18,12 @@ metadata: {{- end }} spec: type: {{ .Values.webhook.serviceType }} + {{- if .Values.webhook.serviceIPFamilyPolicy }} + ipFamilyPolicy: {{ .Values.webhook.serviceIPFamilyPolicy }} + {{- end }} + {{- if .Values.webhook.serviceIPFamilies }} + ipFamilies: {{ .Values.webhook.serviceIPFamilies | toYaml | nindent 2 }} + {{- end }} {{- with .Values.webhook.loadBalancerIP }} loadBalancerIP: {{ . }} {{- end }} diff --git a/internal/constellation/helm/charts/cert-manager/templates/webhook-validating-webhook.yaml b/internal/constellation/helm/charts/cert-manager/templates/webhook-validating-webhook.yaml index a5d168e29..76235fdee 100644 --- a/internal/constellation/helm/charts/cert-manager/templates/webhook-validating-webhook.yaml +++ b/internal/constellation/helm/charts/cert-manager/templates/webhook-validating-webhook.yaml @@ -15,16 +15,10 @@ metadata: {{- end }} webhooks: - name: webhook.cert-manager.io + {{- with .Values.webhook.validatingWebhookConfiguration.namespaceSelector }} namespaceSelector: - matchExpressions: - - key: "cert-manager.io/disable-validation" - operator: "NotIn" - values: - - "true" - - key: "name" - operator: "NotIn" - values: - - {{ include "cert-manager.namespace" . }} + {{- toYaml . | nindent 6 }} + {{- end }} rules: - apiGroups: - "cert-manager.io" diff --git a/internal/constellation/helm/charts/cert-manager/values.yaml b/internal/constellation/helm/charts/cert-manager/values.yaml index 0870178c4..ae304af7b 100644 --- a/internal/constellation/helm/charts/cert-manager/values.yaml +++ b/internal/constellation/helm/charts/cert-manager/values.yaml @@ -1,199 +1,376 @@ +# +docs:section=Global + # Default values for cert-manager. # This is a YAML-formatted file. # Declare variables to be passed into your templates. global: - # Reference to one or more secrets to be used when pulling images - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + # Reference to one or more secrets to be used when pulling images. + # For more information, see [Pull an Image from a Private Registry](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). + # + # For example: + # imagePullSecrets: + # - name: "image-pull-secret" imagePullSecrets: [] - # - name: "image-pull-secret" - - # Labels to apply to all resources + # Labels to apply to all resources. # Please note that this does not add labels to the resources created dynamically by the controllers. # For these resources, you have to add the labels in the template in the cert-manager custom resource: - # eg. podTemplate/ ingressTemplate in ACMEChallengeSolverHTTP01Ingress - # ref: https://cert-manager.io/docs/reference/api-docs/#acme.cert-manager.io/v1.ACMEChallengeSolverHTTP01Ingress - # eg. secretTemplate in CertificateSpec - # ref: https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.CertificateSpec + # For example, podTemplate/ ingressTemplate in ACMEChallengeSolverHTTP01Ingress + # For more information, see the [cert-manager documentation](https://cert-manager.io/docs/reference/api-docs/#acme.cert-manager.io/v1.ACMEChallengeSolverHTTP01Ingress). + # For example, secretTemplate in CertificateSpec + # For more information, see the [cert-manager documentation](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.CertificateSpec). commonLabels: {} - # team_name: dev + # The number of old ReplicaSets to retain to allow rollback (if not set, the default Kubernetes value is set to 10). + # +docs:property + # revisionHistoryLimit: 1 - # Optional priority class to be used for the cert-manager pods + # The optional priority class to be used for the cert-manager pods. priorityClassName: "" rbac: + # Create required ClusterRoles and ClusterRoleBindings for cert-manager. create: true - # Aggregate ClusterRoles to Kubernetes default user-facing roles. Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles + # Aggregate ClusterRoles to Kubernetes default user-facing roles. For more information, see [User-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) aggregateClusterRoles: true podSecurityPolicy: + # Create PodSecurityPolicy for cert-manager. + # + # Note that PodSecurityPolicy was deprecated in Kubernetes 1.21 and removed in Kubernetes 1.25. enabled: false + # Configure the PodSecurityPolicy to use AppArmor. useAppArmor: true - # Set the verbosity of cert-manager. Range of 0 - 6 with 6 being the most verbose. + # Set the verbosity of cert-manager. A range of 0 - 6, with 6 being the most verbose. logLevel: 2 leaderElection: - # Override the namespace used for the leader election lease + # Override the namespace used for the leader election lease. namespace: "kube-system" # The duration that non-leader candidates will wait after observing a # leadership renewal until attempting to acquire leadership of a led but # unrenewed leader slot. This is effectively the maximum duration that a # leader can be stopped before it is replaced by another candidate. + # +docs:property # leaseDuration: 60s # The interval between attempts by the acting master to renew a leadership # slot before it stops leading. This must be less than or equal to the # lease duration. +# +docs:property # renewDeadline: 40s # The duration the clients should wait between attempting acquisition and # renewal of a leadership. +# +docs:property # retryPeriod: 15s + +# This option is equivalent to setting crds.enabled=true and crds.keep=true. +# Deprecated: use crds.enabled and crds.keep instead. installCRDs: false -replicaCount: 1 -strategy: {} -# type: RollingUpdate -# rollingUpdate: -# maxSurge: 0 -# maxUnavailable: 1 - -podDisruptionBudget: +crds: + # This option decides if the CRDs should be installed + # as part of the Helm installation. enabled: false - minAvailable: 1 - # maxUnavailable: 1 -# minAvailable and maxUnavailable can either be set to an integer (e.g. 1) -# or a percentage value (e.g. 25%) + # This option makes it so that the "helm.sh/resource-policy": keep + # annotation is added to the CRD. This will prevent Helm from uninstalling + # the CRD when the Helm release is uninstalled. + # WARNING: when the CRDs are removed, all cert-manager custom resources + # (Certificates, Issuers, ...) will be removed too by the garbage collector. + keep: true +# +docs:section=Controller -# Comma separated list of feature gates that should be enabled on the controller -# Note: do not use this field to pass feature gate values into webhook -# component as this behaviour relies on a bug that will be fixed in cert-manager 1.13 -# https://github.com/cert-manager/cert-manager/pull/6093 -# Use webhook.extraArgs to pass --feature-gates flag directly instead. +# The number of replicas of the cert-manager controller to run. +# +# The default is 1, but in production set this to 2 or 3 to provide high +# availability. +# +# If `replicas > 1`, consider setting `podDisruptionBudget.enabled=true`. +# +# Note that cert-manager uses leader election to ensure that there can +# only be a single instance active at a time. +replicaCount: 1 +# Deployment update strategy for the cert-manager controller deployment. +# For more information, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy). +# +# For example: +# strategy: +# type: RollingUpdate +# rollingUpdate: +# maxSurge: 0 +# maxUnavailable: 1 +strategy: {} +podDisruptionBudget: + # Enable or disable the PodDisruptionBudget resource. + # + # This prevents downtime during voluntary disruptions such as during a Node upgrade. + # For example, the PodDisruptionBudget will block `kubectl drain` + # if it is used on the Node where the only remaining cert-manager + # Pod is currently running. + enabled: false + # This configures the minimum available pods for disruptions. It can either be set to + # an integer (e.g. 1) or a percentage value (e.g. 25%). + # It cannot be used if `maxUnavailable` is set. + # +docs:property + # minAvailable: 1 +# This configures the maximum unavailable pods for disruptions. It can either be set to +# an integer (e.g. 1) or a percentage value (e.g. 25%). +# it cannot be used if `minAvailable` is set. +# +docs:property +# maxUnavailable: 1 + +# A comma-separated list of feature gates that should be enabled on the +# controller pod. featureGates: "" -# The maximum number of challenges that can be scheduled as 'processing' at once +# The maximum number of challenges that can be scheduled as 'processing' at once. maxConcurrentChallenges: 60 image: - repository: quay.io/jetstack/cert-manager-controller - # You can manage a registry with + # The container registry to pull the manager image from. + # +docs:property # registry: quay.io - # repository: jetstack/cert-manager-controller + # The container image for the cert-manager controller. + # +docs:property + repository: quay.io/jetstack/cert-manager-controller # Override the image tag to deploy by setting this variable. - # If no value is set, the chart's appVersion will be used. - # tag: canary + # If no value is set, the chart's appVersion is used. + # +docs:property + # tag: vX.Y.Z - # Setting a digest will override any tag + # Setting a digest will override any tag. + # +docs:property # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 + + # Kubernetes imagePullPolicy on Deployment. pullPolicy: IfNotPresent - digest: sha256:fb2546fe51e49206dbf72bb0d6f909a0018eda0c2b024547b03d3f3d604e4c5e + digest: sha256:9b5d5e9c0fd4944221d059921cc05f388c9a5fc0b02a60b47f0eccfcd8243331 # Override the namespace used to store DNS provider credentials etc. for ClusterIssuer # resources. By default, the same namespace as cert-manager is deployed within is # used. This namespace will not be automatically created by the Helm chart. clusterResourceNamespace: "" -# This namespace allows you to define where the services will be installed into -# if not set then they will use the namespace of the release -# This is helpful when installing cert manager as a chart dependency (sub chart) +# This namespace allows you to define where the services are installed into. +# If not set then they use the namespace of the release. +# This is helpful when installing cert manager as a chart dependency (sub chart). namespace: "" serviceAccount: - # Specifies whether a service account should be created + # Specifies whether a service account should be created. create: true # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # If not set and create is true, a name is generated using the fullname template. + # +docs:property # name: "" - # Optional additional annotations to add to the controller's ServiceAccount + + # Optional additional annotations to add to the controller's Service Account. + # +docs:property # annotations: {} - # Automount API credentials for a Service Account. - # Optional additional labels to add to the controller's ServiceAccount + + # Optional additional labels to add to the controller's Service Account. + # +docs:property # labels: {} + + # Automount API credentials for a Service Account. automountServiceAccountToken: true -# Automounting API credentials for a particular pod +# Automounting API credentials for a particular pod. +# +docs:property # automountServiceAccountToken: true -# When this flag is enabled, secrets will be automatically removed when the certificate resource is deleted +# When this flag is enabled, secrets will be automatically removed when the certificate resource is deleted. enableCertificateOwnerRef: false -# Setting Nameservers for DNS01 Self Check -# See: https://cert-manager.io/docs/configuration/acme/dns01/#setting-nameservers-for-dns01-self-check +# This property is used to configure options for the controller pod. +# This allows setting options that would usually be provided using flags. +# An APIVersion and Kind must be specified in your values.yaml file. +# Flags will override options that are set here. +# +# For example: +# config: +# apiVersion: controller.config.cert-manager.io/v1alpha1 +# kind: ControllerConfiguration +# logging: +# verbosity: 2 +# format: text +# leaderElectionConfig: +# namespace: kube-system +# kubernetesAPIQPS: 9000 +# kubernetesAPIBurst: 9000 +# numberOfConcurrentWorkers: 200 +# featureGates: +# AdditionalCertificateOutputFormats: true +# DisallowInsecureCSRUsageDefinition: true +# ExperimentalCertificateSigningRequestControllers: true +# ExperimentalGatewayAPISupport: true +# LiteralCertificateSubject: true +# SecretsFilteredCaching: true +# ServerSideApply: true +# StableCertificateRequestName: true +# UseCertificateRequestBasicConstraints: true +# ValidateCAA: true +# metricsTLSConfig: +# dynamic: +# secretNamespace: "cert-manager" +# secretName: "cert-manager-metrics-ca" +# dnsNames: +# - cert-manager-metrics +# - cert-manager-metrics.cert-manager +# - cert-manager-metrics.cert-manager.svc +config: {} +# Setting Nameservers for DNS01 Self Check. +# For more information, see the [cert-manager documentation](https://cert-manager.io/docs/configuration/acme/dns01/#setting-nameservers-for-dns01-self-check). -# Comma separated string with host and port of the recursive nameservers cert-manager should query +# A comma-separated string with the host and port of the recursive nameservers cert-manager should query. dns01RecursiveNameservers: "" -# Forces cert-manager to only use the recursive nameservers for verification. -# Enabling this option could cause the DNS01 self check to take longer due to caching performed by the recursive nameservers +# Forces cert-manager to use only the recursive nameservers for verification. +# Enabling this option could cause the DNS01 self check to take longer owing to caching performed by the recursive nameservers. dns01RecursiveNameserversOnly: false +# Option to disable cert-manager's build-in auto-approver. The auto-approver +# approves all CertificateRequests that reference issuers matching the 'approveSignerNames' +# option. This 'disableAutoApproval' option is useful when you want to make all approval decisions +# using a different approver (like approver-policy - https://github.com/cert-manager/approver-policy). +disableAutoApproval: false +# List of signer names that cert-manager will approve by default. CertificateRequests +# referencing these signer names will be auto-approved by cert-manager. Defaults to just +# approving the cert-manager.io Issuer and ClusterIssuer issuers. When set to an empty +# array, ALL issuers will be auto-approved by cert-manager. To disable the auto-approval, +# because eg. you are using approver-policy, you can enable 'disableAutoApproval'. +# ref: https://cert-manager.io/docs/concepts/certificaterequest/#approval +# +docs:property +approveSignerNames: + - issuers.cert-manager.io/* + - clusterissuers.cert-manager.io/* # Additional command line flags to pass to cert-manager controller binary. -# To see all available flags run docker run quay.io/jetstack/cert-manager-controller: --help +# To see all available flags run `docker run quay.io/jetstack/cert-manager-controller: --help`. +# +# Use this flag to enable or disable arbitrary controllers. For example, to disable the CertificiateRequests approver. +# +# For example: +# extraArgs: +# - --controllers=*,-certificaterequests-approver extraArgs: [] -# Use this flag to enable or disable arbitrary controllers, for example, disable the CertificiateRequests approver -# - --controllers=*,-certificaterequests-approver - +# Additional environment variables to pass to cert-manager controller binary. extraEnv: [] # - name: SOME_VAR # value: 'some value' +# Resources to provide to the cert-manager controller pod. +# +# For example: +# requests: +# cpu: 10m +# memory: 32Mi +# +# For more information, see [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). resources: {} -# requests: -# cpu: 10m -# memory: 32Mi - -# Pod Security Context -# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +# Pod Security Context. +# For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). +# +docs:property securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault -# Container Security Context to be set on the controller component container -# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +# Container Security Context to be set on the controller component container. +# For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). +# +docs:property containerSecurityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true + readOnlyRootFilesystem: true +# Additional volumes to add to the cert-manager controller pod. volumes: [] +# Additional volume mounts to add to the cert-manager controller container. volumeMounts: [] -# Optional additional annotations to add to the controller Deployment +# Optional additional annotations to add to the controller Deployment. +# +docs:property # deploymentAnnotations: {} -# Optional additional annotations to add to the controller Pods +# Optional additional annotations to add to the controller Pods. +# +docs:property # podAnnotations: {} + +# Optional additional labels to add to the controller Pods. podLabels: {} -# Optional annotations to add to the controller Service +# Optional annotations to add to the controller Service. +# +docs:property # serviceAnnotations: {} -# Optional additional labels to add to the controller Service +# Optional additional labels to add to the controller Service. +# +docs:property # serviceLabels: {} -# Optional DNS settings, useful if you have a public and private DNS zone for -# the same domain on Route 53. What follows is an example of ensuring +# Optionally set the IP family policy for the controller Service to configure dual-stack; see [Configure dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services). +# +docs:property +# serviceIPFamilyPolicy: "" + +# Optionally set the IP families for the controller Service that should be supported, in the order in which they should be applied to ClusterIP. Can be IPv4 and/or IPv6. +# +docs:property +# serviceIPFamilies: [] + +# Optional DNS settings. These are useful if you have a public and private DNS zone for +# the same domain on Route 53. The following is an example of ensuring # cert-manager can access an ingress or DNS TXT records at all times. -# NOTE: This requires Kubernetes 1.10 or `CustomPodDNS` feature gate enabled for +# Note that this requires Kubernetes 1.10 or `CustomPodDNS` feature gate enabled for # the cluster to work. + +# Pod DNS policy. +# For more information, see [Pod's DNS Policy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy). +# +docs:property # podDnsPolicy: "None" + +# Pod DNS configuration. The podDnsConfig field is optional and can work with any podDnsPolicy +# settings. However, when a Pod's dnsPolicy is set to "None", the dnsConfig field has to be specified. +# For more information, see [Pod's DNS Config](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config). +# +docs:property # podDnsConfig: # nameservers: # - "1.1.1.1" # - "8.8.8.8" + +# Optional hostAliases for cert-manager-controller pods. May be useful when performing ACME DNS-01 self checks. +hostAliases: [] +# - ip: 127.0.0.1 +# hostnames: +# - foo.local +# - bar.local +# - ip: 10.1.2.3 +# hostnames: +# - foo.remote +# - bar.remote + +# The nodeSelector on Pods tells Kubernetes to schedule Pods on the nodes with +# matching labels. +# For more information, see [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). +# +# This default ensures that Pods are only scheduled to Linux nodes. +# It prevents Pods being scheduled to Windows nodes in a mixed OS cluster. +# +docs:property nodeSelector: kubernetes.io/os: linux +# +docs:ignore ingressShim: {} +# Optional default issuer to use for ingress resources. +# +docs:property=ingressShim.defaultIssuerName # defaultIssuerName: "" + +# Optional default issuer kind to use for ingress resources. +# +docs:property=ingressShim.defaultIssuerKind # defaultIssuerKind: "" + +# Optional default issuer group to use for ingress resources. +# +docs:property=ingressShim.defaultIssuerGroup # defaultIssuerGroup: "" -prometheus: - enabled: true - servicemonitor: - enabled: false - prometheusInstance: default - targetPort: 9402 - path: /metrics - interval: 60s - scrapeTimeout: 30s - labels: {} - annotations: {} - honorLabels: false -# Use these variables to configure the HTTP_PROXY environment variables +# Use these variables to configure the HTTP_PROXY environment variables. + +# Configures the HTTP_PROXY environment variable where a HTTP proxy is required. +# +docs:property # http_proxy: "http://proxy:8080" + +# Configures the HTTPS_PROXY environment variable where a HTTP proxy is required. +# +docs:property # https_proxy: "https://proxy:8080" + +# Configures the NO_PROXY environment variable where a HTTP proxy is required, +# but certain domains should be excluded. +# +docs:property # no_proxy: 127.0.0.1,localhost -# A Kubernetes Affinty, if required; see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#affinity-v1-core -# for example: +# A Kubernetes Affinity, if required. For more information, see [Affinity v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#affinity-v1-core). +# +# For example: # affinity: # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -204,16 +381,18 @@ prometheus: # values: # - master affinity: {} -# A list of Kubernetes Tolerations, if required; see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core -# for example: +# A list of Kubernetes Tolerations, if required. For more information, see [Toleration v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core). +# +# For example: # tolerations: # - key: foo.bar.com/role # operator: Equal # value: master # effect: NoSchedule tolerations: [] -# A list of Kubernetes TopologySpreadConstraints, if required; see https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#topologyspreadconstraint-v1-core -# for example: +# A list of Kubernetes TopologySpreadConstraints, if required. For more information, see [Topology spread constraint v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#topologyspreadconstraint-v1-core +# +# For example: # topologySpreadConstraints: # - maxSkew: 2 # topologyKey: topology.kubernetes.io/zone @@ -225,150 +404,361 @@ tolerations: [] topologySpreadConstraints: [] # LivenessProbe settings for the controller container of the controller Pod. # -# Disabled by default, because the controller has a leader election mechanism -# which should cause it to exit if it is unable to renew its leader election -# record. +# This is enabled by default, in order to enable the clock-skew liveness probe that +# restarts the controller in case of a skew between the system clock and the monotonic clock. # LivenessProbe durations and thresholds are based on those used for the Kubernetes -# controller-manager. See: -# https://github.com/kubernetes/kubernetes/blob/806b30170c61a38fedd54cc9ede4cd6275a1ad3b/cmd/kubeadm/app/util/staticpod/utils.go#L241-L245 +# controller-manager. For more information see the following on the +# [Kubernetes GitHub repository](https://github.com/kubernetes/kubernetes/blob/806b30170c61a38fedd54cc9ede4cd6275a1ad3b/cmd/kubeadm/app/util/staticpod/utils.go#L241-L245) +# +docs:property livenessProbe: - enabled: false + enabled: true initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 15 successThreshold: 1 failureThreshold: 8 +# enableServiceLinks indicates whether information about services should be +# injected into the pod's environment variables, matching the syntax of Docker +# links. +enableServiceLinks: false +# +docs:section=Prometheus +prometheus: + # Enable Prometheus monitoring for the cert-manager controller to use with the + # Prometheus Operator. If this option is enabled without enabling `prometheus.servicemonitor.enabled` or + # `prometheus.podmonitor.enabled`, 'prometheus.io' annotations are added to the cert-manager Deployment + # resources. Additionally, a service is created which can be used together + # with your own ServiceMonitor (managed outside of this Helm chart). + # Otherwise, a ServiceMonitor/ PodMonitor is created. + enabled: true + servicemonitor: + # Create a ServiceMonitor to add cert-manager to Prometheus. + enabled: false + # Specifies the `prometheus` label on the created ServiceMonitor. This is + # used when different Prometheus instances have label selectors matching + # different ServiceMonitors. + prometheusInstance: default + # The target port to set on the ServiceMonitor. This must match the port that the + # cert-manager controller is listening on for metrics. + targetPort: 9402 + # The path to scrape for metrics. + path: /metrics + # The interval to scrape metrics. + interval: 60s + # The timeout before a metrics scrape fails. + scrapeTimeout: 30s + # Additional labels to add to the ServiceMonitor. + labels: {} + # Additional annotations to add to the ServiceMonitor. + annotations: {} + # Keep labels from scraped data, overriding server-side labels. + honorLabels: false + # EndpointAdditionalProperties allows setting additional properties on the + # endpoint such as relabelings, metricRelabelings etc. + # + # For example: + # endpointAdditionalProperties: + # relabelings: + # - action: replace + # sourceLabels: + # - __meta_kubernetes_pod_node_name + # targetLabel: instance + # + # +docs:property + endpointAdditionalProperties: {} + # Note that you can not enable both PodMonitor and ServiceMonitor as they are mutually exclusive. Enabling both will result in a error. + podmonitor: + # Create a PodMonitor to add cert-manager to Prometheus. + enabled: false + # Specifies the `prometheus` label on the created PodMonitor. This is + # used when different Prometheus instances have label selectors matching + # different PodMonitors. + prometheusInstance: default + # The path to scrape for metrics. + path: /metrics + # The interval to scrape metrics. + interval: 60s + # The timeout before a metrics scrape fails. + scrapeTimeout: 30s + # Additional labels to add to the PodMonitor. + labels: {} + # Additional annotations to add to the PodMonitor. + annotations: {} + # Keep labels from scraped data, overriding server-side labels. + honorLabels: false + # EndpointAdditionalProperties allows setting additional properties on the + # endpoint such as relabelings, metricRelabelings etc. + # + # For example: + # endpointAdditionalProperties: + # relabelings: + # - action: replace + # sourceLabels: + # - __meta_kubernetes_pod_node_name + # targetLabel: instance + # + # +docs:property + endpointAdditionalProperties: {} +# +docs:section=Webhook webhook: + # Number of replicas of the cert-manager webhook to run. + # + # The default is 1, but in production set this to 2 or 3 to provide high + # availability. + # + # If `replicas > 1`, consider setting `webhook.podDisruptionBudget.enabled=true`. replicaCount: 1 - timeoutSeconds: 10 - # Used to configure options for the webhook pod. - # This allows setting options that'd usually be provided via flags. + # The number of seconds the API server should wait for the webhook to respond before treating the call as a failure. + # The value must be between 1 and 30 seconds. For more information, see + # [Validating webhook configuration v1](https://kubernetes.io/docs/reference/kubernetes-api/extend-resources/validating-webhook-configuration-v1/). + # + # The default is set to the maximum value of 30 seconds as + # users sometimes report that the connection between the K8S API server and + # the cert-manager webhook server times out. + # If *this* timeout is reached, the error message will be "context deadline exceeded", + # which doesn't help the user diagnose what phase of the HTTPS connection timed out. + # For example, it could be during DNS resolution, TCP connection, TLS + # negotiation, HTTP negotiation, or slow HTTP response from the webhook + # server. + # By setting this timeout to its maximum value the underlying timeout error + # message has more chance of being returned to the end user. + timeoutSeconds: 30 + # This is used to configure options for the webhook pod. + # This allows setting options that would usually be provided using flags. # An APIVersion and Kind must be specified in your values.yaml file. - # Flags will override options that are set here. - config: - # apiVersion: webhook.config.cert-manager.io/v1alpha1 - # kind: WebhookConfiguration - - # The port that the webhook should listen on for requests. - # In GKE private clusters, by default kubernetes apiservers are allowed to - # talk to the cluster nodes only on 443 and 10250. so configuring - # securePort: 10250, will work out of the box without needing to add firewall - # rules or requiring NET_BIND_SERVICE capabilities to bind port numbers <1000. - # This should be uncommented and set as a default by the chart once we graduate - # the apiVersion of WebhookConfiguration past v1alpha1. - # securePort: 10250 + # Flags override options that are set here. + # + # For example: + # apiVersion: webhook.config.cert-manager.io/v1alpha1 + # kind: WebhookConfiguration + # # The port that the webhook listens on for requests. + # # In GKE private clusters, by default Kubernetes apiservers are allowed to + # # talk to the cluster nodes only on 443 and 10250. Configuring + # # securePort: 10250 therefore will work out-of-the-box without needing to add firewall + # # rules or requiring NET_BIND_SERVICE capabilities to bind port numbers < 1000. + # # This should be uncommented and set as a default by the chart once + # # the apiVersion of WebhookConfiguration graduates beyond v1alpha1. + # securePort: 10250 + config: {} + # The update strategy for the cert-manager webhook deployment. + # For more information, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) + # + # For example: + # strategy: + # type: RollingUpdate + # rollingUpdate: + # maxSurge: 0 + # maxUnavailable: 1 strategy: {} - # type: RollingUpdate - # rollingUpdate: - # maxSurge: 0 - # maxUnavailable: 1 - - # Pod Security Context to be set on the webhook component Pod - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Pod Security Context to be set on the webhook component Pod. + # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + # +docs:property securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - podDisruptionBudget: - enabled: false - minAvailable: 1 - # maxUnavailable: 1 - # minAvailable and maxUnavailable can either be set to an integer (e.g. 1) - # or a percentage value (e.g. 25%) - - # Container Security Context to be set on the webhook component container - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Container Security Context to be set on the webhook component container. + # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + # +docs:property containerSecurityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # Optional additional annotations to add to the webhook Deployment + readOnlyRootFilesystem: true + podDisruptionBudget: + # Enable or disable the PodDisruptionBudget resource. + # + # This prevents downtime during voluntary disruptions such as during a Node upgrade. + # For example, the PodDisruptionBudget will block `kubectl drain` + # if it is used on the Node where the only remaining cert-manager + # Pod is currently running. + enabled: false + # This property configures the minimum available pods for disruptions. Can either be set to + # an integer (e.g. 1) or a percentage value (e.g. 25%). + # It cannot be used if `maxUnavailable` is set. + # +docs:property + # minAvailable: 1 + # This property configures the maximum unavailable pods for disruptions. Can either be set to + # an integer (e.g. 1) or a percentage value (e.g. 25%). + # It cannot be used if `minAvailable` is set. + # +docs:property + # maxUnavailable: 1 + + # Optional additional annotations to add to the webhook Deployment. + # +docs:property # deploymentAnnotations: {} - # Optional additional annotations to add to the webhook Pods + # Optional additional annotations to add to the webhook Pods. + # +docs:property # podAnnotations: {} - # Optional additional annotations to add to the webhook Service + # Optional additional annotations to add to the webhook Service. + # +docs:property # serviceAnnotations: {} - # Optional additional annotations to add to the webhook MutatingWebhookConfiguration + # Optional additional annotations to add to the webhook MutatingWebhookConfiguration. + # +docs:property # mutatingWebhookConfigurationAnnotations: {} - # Optional additional annotations to add to the webhook ValidatingWebhookConfiguration + # Optional additional annotations to add to the webhook ValidatingWebhookConfiguration. + # +docs:property # validatingWebhookConfigurationAnnotations: {} - + validatingWebhookConfiguration: + # Configure spec.namespaceSelector for validating webhooks. + # +docs:property + namespaceSelector: + matchExpressions: + - key: "cert-manager.io/disable-validation" + operator: "NotIn" + values: + - "true" + mutatingWebhookConfiguration: + # Configure spec.namespaceSelector for mutating webhooks. + # +docs:property + namespaceSelector: {} + # matchLabels: + # key: value + # matchExpressions: + # - key: kubernetes.io/metadata.name + # operator: NotIn + # values: + # - kube-system # Additional command line flags to pass to cert-manager webhook binary. - # To see all available flags run docker run quay.io/jetstack/cert-manager-webhook: --help + # To see all available flags run `docker run quay.io/jetstack/cert-manager-webhook: --help`. extraArgs: [] - # Path to a file containing a WebhookConfiguration object used to configure the webhook + # Path to a file containing a WebhookConfiguration object used to configure the webhook. # - --config= + # Comma separated list of feature gates that should be enabled on the + # webhook pod. + featureGates: "" + # Resources to provide to the cert-manager webhook pod. + # + # For example: + # requests: + # cpu: 10m + # memory: 32Mi + # + # For more information, see [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). resources: {} - # requests: - # cpu: 10m - # memory: 32Mi - - ## Liveness and readiness probe values - ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes - ## + # Liveness probe values. + # For more information, see [Container probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes). + # + # +docs:property livenessProbe: failureThreshold: 3 initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 + # Readiness probe values. + # For more information, see [Container probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes). + # + # +docs:property readinessProbe: failureThreshold: 3 initialDelaySeconds: 5 periodSeconds: 5 successThreshold: 1 timeoutSeconds: 1 + # The nodeSelector on Pods tells Kubernetes to schedule Pods on the nodes with + # matching labels. + # For more information, see [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). + # + # This default ensures that Pods are only scheduled to Linux nodes. + # It prevents Pods being scheduled to Windows nodes in a mixed OS cluster. + # +docs:property nodeSelector: kubernetes.io/os: linux + # A Kubernetes Affinity, if required. For more information, see [Affinity v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#affinity-v1-core). + # + # For example: + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: foo.bar.com/role + # operator: In + # values: + # - master affinity: {} + # A list of Kubernetes Tolerations, if required. For more information, see [Toleration v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core). + # + # For example: + # tolerations: + # - key: foo.bar.com/role + # operator: Equal + # value: master + # effect: NoSchedule tolerations: [] + # A list of Kubernetes TopologySpreadConstraints, if required. For more information, see [Topology spread constraint v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#topologyspreadconstraint-v1-core). + # + # For example: + # topologySpreadConstraints: + # - maxSkew: 2 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: cert-manager + # app.kubernetes.io/component: controller topologySpreadConstraints: [] - # Optional additional labels to add to the Webhook Pods + # Optional additional labels to add to the Webhook Pods. podLabels: {} - # Optional additional labels to add to the Webhook Service + # Optional additional labels to add to the Webhook Service. serviceLabels: {} + # Optionally set the IP family policy for the controller Service to configure dual-stack; see [Configure dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services). + serviceIPFamilyPolicy: "" + # Optionally set the IP families for the controller Service that should be supported, in the order in which they should be applied to ClusterIP. Can be IPv4 and/or IPv6. + serviceIPFamilies: [] image: - repository: quay.io/jetstack/cert-manager-webhook - # You can manage a registry with + # The container registry to pull the webhook image from. + # +docs:property # registry: quay.io - # repository: jetstack/cert-manager-webhook + # The container image for the cert-manager webhook + # +docs:property + repository: quay.io/jetstack/cert-manager-webhook # Override the image tag to deploy by setting this variable. # If no value is set, the chart's appVersion will be used. - # tag: canary + # +docs:property + # tag: vX.Y.Z # Setting a digest will override any tag + # +docs:property # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 + + # Kubernetes imagePullPolicy on Deployment. pullPolicy: IfNotPresent - digest: sha256:db0bb8c02c0b82f3055315fbc52ad41b90fbe94f82431a0d76666f7c6beeb7f0 + digest: sha256:85df7b64a3d66de3cd7995ae0f2151b54fd18db424cb7cf84d3bd6d4a39d975f serviceAccount: - # Specifies whether a service account should be created + # Specifies whether a service account should be created. create: true # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # If not set and create is true, a name is generated using the fullname template. + # +docs:property # name: "" - # Optional additional annotations to add to the controller's ServiceAccount + + # Optional additional annotations to add to the controller's Service Account. + # +docs:property # annotations: {} - # Optional additional labels to add to the webhook's ServiceAccount + + # Optional additional labels to add to the webhook's Service Account. + # +docs:property # labels: {} + # Automount API credentials for a Service Account. automountServiceAccountToken: true - # Automounting API credentials for a particular pod + # Automounting API credentials for a particular pod. + # +docs:property # automountServiceAccountToken: true - # The port that the webhook should listen on for requests. - # In GKE private clusters, by default kubernetes apiservers are allowed to - # talk to the cluster nodes only on 443 and 10250. so configuring - # securePort: 10250, will work out of the box without needing to add firewall - # rules or requiring NET_BIND_SERVICE capabilities to bind port numbers <1000 + # The port that the webhook listens on for requests. + # In GKE private clusters, by default Kubernetes apiservers are allowed to + # talk to the cluster nodes only on 443 and 10250. Configuring + # securePort: 10250, therefore will work out-of-the-box without needing to add firewall + # rules or requiring NET_BIND_SERVICE capabilities to bind port numbers <1000. securePort: 10250 # Specifies if the webhook should be started in hostNetwork mode. # @@ -381,10 +771,12 @@ webhook: # running in hostNetwork mode. hostNetwork: false # Specifies how the service should be handled. Useful if you want to expose the - # webhook to outside of the cluster. In some cases, the control plane cannot + # webhook outside of the cluster. In some cases, the control plane cannot # reach internal services. serviceType: ClusterIP - # loadBalancerIP: + # Specify the load balancer IP for the created service. + # +docs:property + # loadBalancerIP: "10.10.10.10" # Overrides the mutating webhook and validating webhook so they reach the webhook # service using the `url` field instead of a service. @@ -393,11 +785,18 @@ webhook: # Enables default network policies for webhooks. networkPolicy: + # Create network policies for the webhooks. enabled: false + # Ingress rule for the webhook network policy. By default, it allows all + # inbound traffic. + # +docs:property ingress: - from: - ipBlock: cidr: 0.0.0.0/0 + # Egress rule for the webhook network policy. By default, it allows all + # outbound traffic to ports 80 and 443, as well as DNS ports. + # +docs:property egress: - ports: - port: 80 @@ -408,202 +807,393 @@ webhook: protocol: TCP - port: 53 protocol: UDP - # On OpenShift and OKD, the Kubernetes API server listens on + # On OpenShift and OKD, the Kubernetes API server listens on. # port 6443. - port: 6443 protocol: TCP to: - ipBlock: cidr: 0.0.0.0/0 + # Additional volumes to add to the cert-manager controller pod. volumes: [] + # Additional volume mounts to add to the cert-manager controller container. volumeMounts: [] + # enableServiceLinks indicates whether information about services should be + # injected into the pod's environment variables, matching the syntax of Docker + # links. + enableServiceLinks: false +# +docs:section=CA Injector cainjector: + # Create the CA Injector deployment enabled: true + # The number of replicas of the cert-manager cainjector to run. + # + # The default is 1, but in production set this to 2 or 3 to provide high + # availability. + # + # If `replicas > 1`, consider setting `cainjector.podDisruptionBudget.enabled=true`. + # + # Note that cert-manager uses leader election to ensure that there can + # only be a single instance active at a time. replicaCount: 1 + # This is used to configure options for the cainjector pod. + # It allows setting options that are usually provided via flags. + # An APIVersion and Kind must be specified in your values.yaml file. + # Flags override options that are set here. + # + # For example: + # apiVersion: cainjector.config.cert-manager.io/v1alpha1 + # kind: CAInjectorConfiguration + # logging: + # verbosity: 2 + # format: text + # leaderElectionConfig: + # namespace: kube-system + config: {} + # Deployment update strategy for the cert-manager cainjector deployment. + # For more information, see the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy). + # + # For example: + # strategy: + # type: RollingUpdate + # rollingUpdate: + # maxSurge: 0 + # maxUnavailable: 1 strategy: {} - # type: RollingUpdate - # rollingUpdate: - # maxSurge: 0 - # maxUnavailable: 1 - # Pod Security Context to be set on the cainjector component Pod - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + # +docs:property securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - podDisruptionBudget: - enabled: false - minAvailable: 1 - # maxUnavailable: 1 - # minAvailable and maxUnavailable can either be set to an integer (e.g. 1) - # or a percentage value (e.g. 25%) - # Container Security Context to be set on the cainjector component container - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + # +docs:property containerSecurityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # Optional additional annotations to add to the cainjector Deployment + readOnlyRootFilesystem: true + podDisruptionBudget: + # Enable or disable the PodDisruptionBudget resource. + # + # This prevents downtime during voluntary disruptions such as during a Node upgrade. + # For example, the PodDisruptionBudget will block `kubectl drain` + # if it is used on the Node where the only remaining cert-manager + # Pod is currently running. + enabled: false + # `minAvailable` configures the minimum available pods for disruptions. It can either be set to + # an integer (e.g. 1) or a percentage value (e.g. 25%). + # Cannot be used if `maxUnavailable` is set. + # +docs:property + # minAvailable: 1 + # `maxUnavailable` configures the maximum unavailable pods for disruptions. It can either be set to + # an integer (e.g. 1) or a percentage value (e.g. 25%). + # Cannot be used if `minAvailable` is set. + # +docs:property + # maxUnavailable: 1 + + # Optional additional annotations to add to the cainjector Deployment. + # +docs:property # deploymentAnnotations: {} - # Optional additional annotations to add to the cainjector Pods + # Optional additional annotations to add to the cainjector Pods. + # +docs:property # podAnnotations: {} # Additional command line flags to pass to cert-manager cainjector binary. - # To see all available flags run docker run quay.io/jetstack/cert-manager-cainjector: --help + # To see all available flags run `docker run quay.io/jetstack/cert-manager-cainjector: --help`. extraArgs: [] - # Enable profiling for cainjector + # Enable profiling for cainjector. # - --enable-profiling=true + # Comma separated list of feature gates that should be enabled on the + # cainjector pod. + featureGates: "" + # Resources to provide to the cert-manager cainjector pod. + # + # For example: + # requests: + # cpu: 10m + # memory: 32Mi + # + # For more information, see [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). resources: {} - # requests: - # cpu: 10m - # memory: 32Mi - + # The nodeSelector on Pods tells Kubernetes to schedule Pods on the nodes with + # matching labels. + # For more information, see [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). + # + # This default ensures that Pods are only scheduled to Linux nodes. + # It prevents Pods being scheduled to Windows nodes in a mixed OS cluster. + # +docs:property nodeSelector: kubernetes.io/os: linux + # A Kubernetes Affinity, if required. For more information, see [Affinity v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#affinity-v1-core). + # + # For example: + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: foo.bar.com/role + # operator: In + # values: + # - master affinity: {} + # A list of Kubernetes Tolerations, if required. For more information, see [Toleration v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core). + # + # For example: + # tolerations: + # - key: foo.bar.com/role + # operator: Equal + # value: master + # effect: NoSchedule tolerations: [] + # A list of Kubernetes TopologySpreadConstraints, if required. For more information, see [Topology spread constraint v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#topologyspreadconstraint-v1-core). + # + # For example: + # topologySpreadConstraints: + # - maxSkew: 2 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: cert-manager + # app.kubernetes.io/component: controller topologySpreadConstraints: [] - # Optional additional labels to add to the CA Injector Pods + # Optional additional labels to add to the CA Injector Pods. podLabels: {} image: - repository: quay.io/jetstack/cert-manager-cainjector - # You can manage a registry with + # The container registry to pull the cainjector image from. + # +docs:property # registry: quay.io - # repository: jetstack/cert-manager-cainjector + # The container image for the cert-manager cainjector + # +docs:property + repository: quay.io/jetstack/cert-manager-cainjector # Override the image tag to deploy by setting this variable. # If no value is set, the chart's appVersion will be used. - # tag: canary + # +docs:property + # tag: vX.Y.Z - # Setting a digest will override any tag + # Setting a digest will override any tag. + # +docs:property # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 + + # Kubernetes imagePullPolicy on Deployment. pullPolicy: IfNotPresent - digest: sha256:2a70d9497a645101210d077874c35dc0431233d8c6e53a851835ca301523d64b + digest: sha256:edb1c1e0083ee4cd8e2ccb296ee0f436d2e465ecf90159f9d03141fc19bd3c23 serviceAccount: - # Specifies whether a service account should be created + # Specifies whether a service account should be created. create: true # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template + # +docs:property # name: "" - # Optional additional annotations to add to the controller's ServiceAccount + + # Optional additional annotations to add to the controller's Service Account. + # +docs:property # annotations: {} - # Automount API credentials for a Service Account. - # Optional additional labels to add to the cainjector's ServiceAccount + + # Optional additional labels to add to the cainjector's Service Account. + # +docs:property # labels: {} + + # Automount API credentials for a Service Account. automountServiceAccountToken: true - # Automounting API credentials for a particular pod + # Automounting API credentials for a particular pod. + # +docs:property # automountServiceAccountToken: true + + # Additional volumes to add to the cert-manager controller pod. volumes: [] + # Additional volume mounts to add to the cert-manager controller container. volumeMounts: [] + # enableServiceLinks indicates whether information about services should be + # injected into the pod's environment variables, matching the syntax of Docker + # links. + enableServiceLinks: false +# +docs:section=ACME Solver acmesolver: image: - repository: quay.io/jetstack/cert-manager-acmesolver - # You can manage a registry with + # The container registry to pull the acmesolver image from. + # +docs:property # registry: quay.io - # repository: jetstack/cert-manager-acmesolver - digest: sha256:12a62e54ba8defda94df71ef76f9c8fe68405d59370f665991734d6b692e35f2 -# Override the image tag to deploy by setting this variable. -# If no value is set, the chart's appVersion will be used. -# tag: canary + # The container image for the cert-manager acmesolver. + # +docs:property + repository: quay.io/jetstack/cert-manager-acmesolver + # Override the image tag to deploy by setting this variable. + # If no value is set, the chart's appVersion is used. + # +docs:property + # tag: vX.Y.Z -# Setting a digest will override any tag -# digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 + # Setting a digest will override any tag. + # +docs:property + # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 + # Kubernetes imagePullPolicy on Deployment. + pullPolicy: IfNotPresent + digest: sha256:99feb5d6cd8e8b4c6eb1ab14b317304141c25114b0bd3e5588b9f551f664cb8f +# +docs:section=Startup API Check # This startupapicheck is a Helm post-install hook that waits for the webhook # endpoints to become available. -# The check is implemented using a Kubernetes Job- if you are injecting mesh -# sidecar proxies into cert-manager pods, you probably want to ensure that they -# are not injected into this Job's pod. Otherwise the installation may time out -# due to the Job never being completed because the sidecar proxy does not exit. -# See https://github.com/cert-manager/cert-manager/pull/4414 for context. +# The check is implemented using a Kubernetes Job - if you are injecting mesh +# sidecar proxies into cert-manager pods, ensure that they +# are not injected into this Job's pod. Otherwise, the installation may time out +# owing to the Job never being completed because the sidecar proxy does not exit. +# For more information, see [this note](https://github.com/cert-manager/cert-manager/pull/4414). startupapicheck: + # Enables the startup api check. enabled: true - # Pod Security Context to be set on the startupapicheck component Pod - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Pod Security Context to be set on the startupapicheck component Pod. + # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + # +docs:property securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - # Container Security Context to be set on the controller component container - # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Container Security Context to be set on the controller component container. + # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + # +docs:property containerSecurityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # Timeout for 'kubectl check api' command + readOnlyRootFilesystem: true + # Timeout for 'kubectl check api' command. timeout: 1m # Job backoffLimit backoffLimit: 4 - # Optional additional annotations to add to the startupapicheck Job + # Optional additional annotations to add to the startupapicheck Job. + # +docs:property jobAnnotations: helm.sh/hook: post-install helm.sh/hook-weight: "1" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - # Optional additional annotations to add to the startupapicheck Pods + # Optional additional annotations to add to the startupapicheck Pods. + # +docs:property # podAnnotations: {} # Additional command line flags to pass to startupapicheck binary. - # To see all available flags run docker run quay.io/jetstack/cert-manager-ctl: --help - extraArgs: [] + # To see all available flags run `docker run quay.io/jetstack/cert-manager-startupapicheck: --help`. + # + # Verbose logging is enabled by default so that if startupapicheck fails, you + # can know what exactly caused the failure. Verbose logs include details of + # the webhook URL, IP address and TCP connect errors for example. + # +docs:property + extraArgs: + - -v + # Resources to provide to the cert-manager controller pod. + # + # For example: + # requests: + # cpu: 10m + # memory: 32Mi + # + # For more information, see [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). resources: {} - # requests: - # cpu: 10m - # memory: 32Mi - + # The nodeSelector on Pods tells Kubernetes to schedule Pods on the nodes with + # matching labels. + # For more information, see [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). + # + # This default ensures that Pods are only scheduled to Linux nodes. + # It prevents Pods being scheduled to Windows nodes in a mixed OS cluster. + # +docs:property nodeSelector: kubernetes.io/os: linux + # A Kubernetes Affinity, if required. For more information, see [Affinity v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#affinity-v1-core). + # For example: + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: foo.bar.com/role + # operator: In + # values: + # - master affinity: {} + # A list of Kubernetes Tolerations, if required. For more information, see [Toleration v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core). + # + # For example: + # tolerations: + # - key: foo.bar.com/role + # operator: Equal + # value: master + # effect: NoSchedule tolerations: [] - # Optional additional labels to add to the startupapicheck Pods + # Optional additional labels to add to the startupapicheck Pods. podLabels: {} image: - repository: quay.io/jetstack/cert-manager-ctl - # You can manage a registry with + # The container registry to pull the startupapicheck image from. + # +docs:property # registry: quay.io - # repository: jetstack/cert-manager-ctl + # The container image for the cert-manager startupapicheck. + # +docs:property + repository: quay.io/jetstack/cert-manager-startupapicheck # Override the image tag to deploy by setting this variable. - # If no value is set, the chart's appVersion will be used. - # tag: canary + # If no value is set, the chart's appVersion is used. + # +docs:property + # tag: vX.Y.Z - # Setting a digest will override any tag + # Setting a digest will override any tag. + # +docs:property # digest: sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 + + # Kubernetes imagePullPolicy on Deployment. pullPolicy: IfNotPresent - digest: sha256:1b988a4a2ae83aae995d396fa67fdb4c90bc55bc91ea74679f17c6c347541406 + digest: sha256:6365e940a5a913a3aeca0ea519102236d9bec5f0e8f0011fa3498c26d18348e5 rbac: - # annotations for the startup API Check job RBAC and PSP resources + # annotations for the startup API Check job RBAC and PSP resources. + # +docs:property annotations: helm.sh/hook: post-install helm.sh/hook-weight: "-5" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded - # Automounting API credentials for a particular pod + # Automounting API credentials for a particular pod. + # +docs:property # automountServiceAccountToken: true serviceAccount: - # Specifies whether a service account should be created + # Specifies whether a service account should be created. create: true # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # If not set and create is true, a name is generated using the fullname template. + # +docs:property # name: "" - # Optional additional annotations to add to the Job's ServiceAccount + # Optional additional annotations to add to the Job's Service Account. + # +docs:property annotations: helm.sh/hook: post-install helm.sh/hook-weight: "-5" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded # Automount API credentials for a Service Account. + # +docs:property automountServiceAccountToken: true - # Optional additional labels to add to the startupapicheck's ServiceAccount + # Optional additional labels to add to the startupapicheck's Service Account. + # +docs:property # labels: {} + # Additional volumes to add to the cert-manager controller pod. volumes: [] + # Additional volume mounts to add to the cert-manager controller container. volumeMounts: [] + # enableServiceLinks indicates whether information about services should be + # injected into pod's environment variables, matching the syntax of Docker + # links. + enableServiceLinks: false +# Create dynamic manifests via values. +# +# For example: +# extraObjects: +# - | +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: '{{ template "cert-manager.name" . }}-extra-configmap' +extraObjects: [] diff --git a/internal/constellation/helm/charts/cilium/Chart.yaml b/internal/constellation/helm/charts/cilium/Chart.yaml index f4a2fe998..cca1abd89 100644 --- a/internal/constellation/helm/charts/cilium/Chart.yaml +++ b/internal/constellation/helm/charts/cilium/Chart.yaml @@ -2,10 +2,10 @@ apiVersion: v2 name: cilium displayName: Cilium home: https://cilium.io/ -version: 1.15.0-pre.2 -appVersion: 1.15.0-pre.2 +version: 1.15.19-edg.0 +appVersion: 1.15.19-edg.0 kubeVersion: ">= 1.16.0-0" -icon: https://cdn.jsdelivr.net/gh/cilium/cilium@main/Documentation/images/logo-solo.svg +icon: https://cdn.jsdelivr.net/gh/cilium/cilium@v1.15/Documentation/images/logo-solo.svg description: eBPF-based Networking, Security, and Observability keywords: - BPF diff --git a/internal/constellation/helm/charts/cilium/README.md b/internal/constellation/helm/charts/cilium/README.md index 615f46753..5e177569d 100644 --- a/internal/constellation/helm/charts/cilium/README.md +++ b/internal/constellation/helm/charts/cilium/README.md @@ -1,6 +1,6 @@ # cilium -![Version: 1.15.0-pre.2](https://img.shields.io/badge/Version-1.15.0--pre.2-informational?style=flat-square) ![AppVersion: 1.15.0-pre.2](https://img.shields.io/badge/AppVersion-1.15.0--pre.2-informational?style=flat-square) +![Version: 1.15.19](https://img.shields.io/badge/Version-1.15.19-informational?style=flat-square) ![AppVersion: 1.15.19](https://img.shields.io/badge/AppVersion-1.15.19-informational?style=flat-square) Cilium is open source software for providing and transparently securing network connectivity and loadbalancing between application workloads such as @@ -46,7 +46,7 @@ offer from the [Getting Started Guides page](https://docs.cilium.io/en/stable/ge ## Getting Help The best way to get help if you get stuck is to ask a question on the -[Cilium Slack channel](https://cilium.herokuapp.com/). With Cilium +[Cilium Slack channel](https://slack.cilium.io). With Cilium contributors across the globe, there is almost always someone available to help. ## Values @@ -71,14 +71,19 @@ contributors across the globe, there is almost always someone available to help. | authentication.mutual.spire.annotations | object | `{}` | Annotations to be added to all top-level spire objects (resources under templates/spire) | | authentication.mutual.spire.connectionTimeout | string | `"30s"` | SPIRE connection timeout | | authentication.mutual.spire.enabled | bool | `false` | Enable SPIRE integration (beta) | +| authentication.mutual.spire.install.agent.affinity | object | `{}` | SPIRE agent affinity configuration | | authentication.mutual.spire.install.agent.annotations | object | `{}` | SPIRE agent annotations | -| authentication.mutual.spire.install.agent.image | object | `{"digest":"sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823","override":null,"pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-agent","tag":"1.6.3","useDigest":true}` | SPIRE agent image | +| authentication.mutual.spire.install.agent.image | object | `{"digest":"sha256:99405637647968245ff9fe215f8bd2bd0ea9807be9725f8bf19fe1b21471e52b","override":null,"pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-agent","tag":"1.8.5","useDigest":true}` | SPIRE agent image | | authentication.mutual.spire.install.agent.labels | object | `{}` | SPIRE agent labels | +| authentication.mutual.spire.install.agent.nodeSelector | object | `{}` | SPIRE agent nodeSelector configuration ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| authentication.mutual.spire.install.agent.podSecurityContext | object | `{}` | Security context to be added to spire agent pods. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| authentication.mutual.spire.install.agent.securityContext | object | `{}` | Security context to be added to spire agent containers. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container | | authentication.mutual.spire.install.agent.serviceAccount | object | `{"create":true,"name":"spire-agent"}` | SPIRE agent service account | | authentication.mutual.spire.install.agent.skipKubeletVerification | bool | `true` | SPIRE Workload Attestor kubelet verification. | -| authentication.mutual.spire.install.agent.tolerations | list | `[]` | SPIRE agent tolerations configuration ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| authentication.mutual.spire.install.agent.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/not-ready"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"},{"key":"CriticalAddonsOnly","operator":"Exists"}]` | SPIRE agent tolerations configuration By default it follows the same tolerations as the agent itself to allow the Cilium agent on this node to connect to SPIRE. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | authentication.mutual.spire.install.enabled | bool | `true` | Enable SPIRE installation. This will only take effect only if authentication.mutual.spire.enabled is true | -| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.35.0","useDigest":true}` | init container image of SPIRE agent and server | +| authentication.mutual.spire.install.existingNamespace | bool | `false` | SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. | +| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:7edf5efe6b86dbf01ccc3c76b32a37a8e23b84e6bad81ce8ae8c221fa456fda8","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.36.1","useDigest":true}` | init container image of SPIRE agent and server | | authentication.mutual.spire.install.namespace | string | `"cilium-spire"` | SPIRE namespace to install into | | authentication.mutual.spire.install.server.affinity | object | `{}` | SPIRE server affinity configuration | | authentication.mutual.spire.install.server.annotations | object | `{}` | SPIRE server annotations | @@ -88,7 +93,7 @@ contributors across the globe, there is almost always someone available to help. | authentication.mutual.spire.install.server.dataStorage.enabled | bool | `true` | Enable SPIRE server data storage | | authentication.mutual.spire.install.server.dataStorage.size | string | `"1Gi"` | Size of the SPIRE server data storage | | authentication.mutual.spire.install.server.dataStorage.storageClass | string | `nil` | StorageClass of the SPIRE server data storage | -| authentication.mutual.spire.install.server.image | object | `{"digest":"sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f","override":null,"pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-server","tag":"1.6.3","useDigest":true}` | SPIRE server image | +| authentication.mutual.spire.install.server.image | object | `{"digest":"sha256:28269265882048dcf0fed32fe47663cd98613727210b8d1a55618826f9bf5428","override":null,"pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-server","tag":"1.8.5","useDigest":true}` | SPIRE server image | | authentication.mutual.spire.install.server.initContainers | list | `[]` | SPIRE server init containers | | authentication.mutual.spire.install.server.labels | object | `{}` | SPIRE server labels | | authentication.mutual.spire.install.server.nodeSelector | object | `{}` | SPIRE server nodeSelector configuration ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | @@ -112,11 +117,11 @@ contributors across the globe, there is almost always someone available to help. | bgp.announce.loadbalancerIP | bool | `false` | Enable allocation and announcement of service LoadBalancer IPs | | bgp.announce.podCIDR | bool | `false` | Enable announcement of node pod CIDR | | bgp.enabled | bool | `false` | Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside cilium-agent and cilium-operator | -| bgpControlPlane | object | `{"enabled":false,"secretsNamespace":{"create":true,"name":"cilium-bgp-secrets"}}` | This feature set enables virtual BGP routers to be created via CiliumBGPPeeringPolicy CRDs. | +| bgpControlPlane | object | `{"enabled":false,"secretsNamespace":{"create":false,"name":"kube-system"}}` | This feature set enables virtual BGP routers to be created via CiliumBGPPeeringPolicy CRDs. | | bgpControlPlane.enabled | bool | `false` | Enables the BGP control plane. | -| bgpControlPlane.secretsNamespace | object | `{"create":true,"name":"cilium-bgp-secrets"}` | SecretsNamespace is the namespace which BGP support will retrieve secrets from. | -| bgpControlPlane.secretsNamespace.create | bool | `true` | Create secrets namespace for BGP secrets. | -| bgpControlPlane.secretsNamespace.name | string | `"cilium-bgp-secrets"` | The name of the secret namespace to which Cilium agents are given read access | +| bgpControlPlane.secretsNamespace | object | `{"create":false,"name":"kube-system"}` | SecretsNamespace is the namespace which BGP support will retrieve secrets from. | +| bgpControlPlane.secretsNamespace.create | bool | `false` | Create secrets namespace for BGP secrets. | +| bgpControlPlane.secretsNamespace.name | string | `"kube-system"` | The name of the secret namespace to which Cilium agents are given read access | | bpf.authMapMax | int | `524288` | Configure the maximum number of entries in auth map. | | bpf.autoMount.enabled | bool | `true` | Enable automatic mount of BPF filesystem When `autoMount` is enabled, the BPF filesystem is mounted at `bpf.root` path on the underlying host and inside the cilium agent pod. If users disable `autoMount`, it's expected that users have mounted bpffs filesystem at the specified `bpf.root` volume, and then the volume will be mounted inside the cilium agent pod at the same path. | | bpf.ctAnyMax | int | `262144` | Configure the maximum number of entries for the non-TCP connection tracking table. | @@ -131,13 +136,14 @@ contributors across the globe, there is almost always someone available to help. | bpf.monitorInterval | string | `"5s"` | Configure the typical time between monitor notifications for active connections. | | bpf.natMax | int | `524288` | Configure the maximum number of entries for the NAT table. | | bpf.neighMax | int | `524288` | Configure the maximum number of entries for the neighbor table. | -| bpf.policyMapMax | int | `16384` | Configure the maximum number of entries in endpoint policy map (per endpoint). | +| bpf.nodeMapMax | int | `nil` | Configures the maximum number of entries for the node table. | +| bpf.policyMapMax | int | `16384` | Configure the maximum number of entries in endpoint policy map (per endpoint). @schema type: [null, integer] @schema | | bpf.preallocateMaps | bool | `false` | Enables pre-allocation of eBPF map values. This increases memory usage but can reduce latency. | | bpf.root | string | `"/sys/fs/bpf"` | Configure the mount point for the BPF filesystem | | bpf.tproxy | bool | `false` | Configure the eBPF-based TPROXY to reduce reliance on iptables rules for implementing Layer 7 policy. | | bpf.vlanBypass | list | `[]` | Configure explicitly allowed VLAN id's for bpf logic bypass. [0] will allow all VLAN id's without any filtering. | | bpfClockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | -| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"extraVolumeMounts":[],"extraVolumes":[],"image":{"digest":"sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.1.9","useDigest":true},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"extraVolumeMounts":[],"extraVolumes":[],"image":{"digest":"sha256:28511366bb5dc99b6ec424dc87399945714d57a586194658d9e2316ba3db4d04","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.1.19","useDigest":true},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | | certgen.affinity | object | `{}` | Affinity for certgen | | certgen.annotations | object | `{"cronJob":{},"job":{}}` | Annotations to be added to the hubble-certgen initial Job and CronJob | | certgen.extraVolumeMounts | list | `[]` | Additional certgen volumeMounts. | @@ -155,7 +161,8 @@ contributors across the globe, there is almost always someone available to help. | cluster.name | string | `"default"` | Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. | | clustermesh.annotations | object | `{}` | Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) | | clustermesh.apiserver.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"clustermesh-apiserver"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for clustermesh.apiserver | -| clustermesh.apiserver.etcd.image | object | `{"digest":"sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/coreos/etcd","tag":"v3.5.4","useDigest":true}` | Clustermesh API server etcd image. | +| clustermesh.apiserver.etcd.init.extraArgs | list | `[]` | Additional arguments to `clustermesh-apiserver etcdinit`. | +| clustermesh.apiserver.etcd.init.extraEnv | list | `[]` | Additional environment variables to `clustermesh-apiserver etcdinit`. | | clustermesh.apiserver.etcd.init.resources | object | `{}` | Specifies the resources for etcd init container in the apiserver | | clustermesh.apiserver.etcd.lifecycle | object | `{}` | lifecycle setting for the etcd container | | clustermesh.apiserver.etcd.resources | object | `{}` | Specifies the resources for etcd container in the apiserver | @@ -164,12 +171,11 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.apiserver.extraEnv | list | `[]` | Additional clustermesh-apiserver environment variables. | | clustermesh.apiserver.extraVolumeMounts | list | `[]` | Additional clustermesh-apiserver volumeMounts. | | clustermesh.apiserver.extraVolumes | list | `[]` | Additional clustermesh-apiserver volumes. | -| clustermesh.apiserver.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.15.0-pre.2","useDigest":false}` | Clustermesh API server image. | +| clustermesh.apiserver.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.15.19","useDigest":false}` | Clustermesh API server image. | | clustermesh.apiserver.kvstoremesh.enabled | bool | `false` | Enable KVStoreMesh. KVStoreMesh caches the information retrieved from the remote clusters in the local etcd instance. | | clustermesh.apiserver.kvstoremesh.extraArgs | list | `[]` | Additional KVStoreMesh arguments. | | clustermesh.apiserver.kvstoremesh.extraEnv | list | `[]` | Additional KVStoreMesh environment variables. | | clustermesh.apiserver.kvstoremesh.extraVolumeMounts | list | `[]` | Additional KVStoreMesh volumeMounts. | -| clustermesh.apiserver.kvstoremesh.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/kvstoremesh","tag":"v1.15.0-pre.2","useDigest":false}` | KVStoreMesh image. | | clustermesh.apiserver.kvstoremesh.lifecycle | object | `{}` | lifecycle setting for the KVStoreMesh container | | clustermesh.apiserver.kvstoremesh.resources | object | `{}` | Resource requests and limits for the KVStoreMesh container | | clustermesh.apiserver.kvstoremesh.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | KVStoreMesh Security context | @@ -207,6 +213,8 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.apiserver.service.annotations | object | `{}` | Annotations for the clustermesh-apiserver For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 | | clustermesh.apiserver.service.externalTrafficPolicy | string | `nil` | The externalTrafficPolicy of service used for apiserver access. | | clustermesh.apiserver.service.internalTrafficPolicy | string | `nil` | The internalTrafficPolicy of service used for apiserver access. | +| clustermesh.apiserver.service.loadBalancerClass | string | `nil` | Configure a loadBalancerClass. Allows to configure the loadBalancerClass on the clustermesh-apiserver LB service in case the Service type is set to LoadBalancer (requires Kubernetes 1.24+). | +| clustermesh.apiserver.service.loadBalancerIP | string | `nil` | Configure a specific loadBalancerIP. Allows to configure a specific loadBalancerIP on the clustermesh-apiserver LB service in case the Service type is set to LoadBalancer. | | clustermesh.apiserver.service.nodePort | int | `32379` | Optional port to use as the node port for apiserver access. WARNING: make sure to configure a different NodePort in each cluster if kube-proxy replacement is enabled, as Cilium is currently affected by a known bug (#24692) when NodePorts are handled by the KPR implementation. If a service with the same NodePort exists both in the local and the remote cluster, all traffic originating from inside the cluster and targeting the corresponding NodePort will be redirected to a local backend, regardless of whether the destination node belongs to the local or the remote cluster. | | clustermesh.apiserver.service.type | string | `"NodePort"` | The type of service used for apiserver access. | | clustermesh.apiserver.terminationGracePeriodSeconds | int | `30` | terminationGracePeriodSeconds for the clustermesh-apiserver deployment | @@ -228,6 +236,7 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.config.clusters | list | `[]` | List of clusters to be peered in the mesh. | | clustermesh.config.domain | string | `"mesh.cilium.io"` | Default dns domain for the Clustermesh API servers This is used in the case cluster addresses are not provided and IPs are used. | | clustermesh.config.enabled | bool | `false` | Enable the Clustermesh explicit configuration. | +| clustermesh.maxConnectedClusters | int | `255` | The maximum number of clusters to support in a ClusterMesh. This value cannot be changed on running clusters, and all clusters in a ClusterMesh must be configured with the same value. Values > 255 will decrease the maximum allocatable cluster-local identities. Supported values are 255 and 511. | | clustermesh.useAPIServer | bool | `false` | Deploy clustermesh-apiserver for clustermesh | | cni.binPath | string | `"/opt/cni/bin"` | Configure the path to the CNI binary directory on the host. | | cni.chainingMode | string | `nil` | Configure chaining on top of other CNI plugins. Possible values: - none - aws-cni - flannel - generic-veth - portmap | @@ -240,6 +249,7 @@ contributors across the globe, there is almost always someone available to help. | cni.hostConfDirMountPath | string | `"/host/etc/cni/net.d"` | Configure the path to where the CNI configuration directory is mounted inside the agent pod. | | cni.install | bool | `true` | Install the CNI configuration and binary files into the filesystem. | | cni.logFile | string | `"/var/run/cilium/cilium-cni.log"` | Configure the log file for CNI logging with retention policy of 7 days. Disable CNI file logging by setting this field to empty explicitly. | +| cni.resources | object | `{"requests":{"cpu":"100m","memory":"10Mi"}}` | Specifies the resources for the cni initContainer | | cni.uninstall | bool | `false` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | | conntrackGCInterval | string | `"0s"` | Configure how frequently garbage collection should occur for the datapath connection tracking table. | | conntrackGCMaxInterval | string | `""` | Configure the maximum frequency for the garbage collection of the connection tracking table. Only affects the automatic computation for the frequency and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently clean up unused identities created from ToFQDN policies. | @@ -266,17 +276,16 @@ contributors across the globe, there is almost always someone available to help. | dnsProxy.preCache | string | `""` | DNS cache data at this path is preloaded on agent startup. | | dnsProxy.proxyPort | int | `0` | Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. | | dnsProxy.proxyResponseMaxDelay | string | `"100ms"` | The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. | +| dnsProxy.socketLingerTimeout | int | `10` | Timeout (in seconds) when closing the connection between the DNS proxy and the upstream server. If set to 0, the connection is closed immediately (with TCP RST). If set to -1, the connection is closed asynchronously in the background. | | egressGateway.enabled | bool | `false` | Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. | | egressGateway.installRoutes | bool | `false` | Deprecated without a replacement necessary. | | egressGateway.reconciliationTriggerInterval | string | `"1s"` | Time between triggers of egress gateway state reconciliations | | enableCiliumEndpointSlice | bool | `false` | Enable CiliumEndpointSlice feature. | -| enableCnpStatusUpdates | bool | `false` | Whether to enable CNP status updates. | | enableCriticalPriorityClass | bool | `true` | Explicitly enable or disable priority class. .Capabilities.KubeVersion is unsettable in `helm template` calls, it depends on k8s libraries version that Helm was compiled against. This option allows to explicitly disable setting the priority class, which is useful for rendering charts for gke clusters in advance. | | enableIPv4BIGTCP | bool | `false` | Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods | | enableIPv4Masquerade | bool | `true` | Enables masquerading of IPv4 traffic leaving the node from endpoints. | | enableIPv6BIGTCP | bool | `false` | Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods | | enableIPv6Masquerade | bool | `true` | Enables masquerading of IPv6 traffic leaving the node from endpoints. | -| enableK8sEventHandover | bool | `false` | Configures the use of the KVStore to optimize Kubernetes event handling by mirroring it into the KVstore for reduced overhead in large clusters. | | enableK8sTerminatingEndpoint | bool | `true` | Configure whether to enable auto detect of terminating state for endpoints in order to support graceful termination. | | enableMasqueradeRouteSource | bool | `false` | Enables masquerading to the source of the route for traffic leaving the node from endpoints. | | enableRuntimeDeviceDetection | bool | `false` | Enables experimental support for the detection of new and removed datapath devices. When devices change the eBPF datapath is reloaded and services updated. If "devices" is set then only those devices, or devices matching a wildcard will be considered. | @@ -293,10 +302,11 @@ contributors across the globe, there is almost always someone available to help. | encryption.mountPath | string | `"/etc/ipsec"` | Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. Path to mount the secret inside the Cilium pod. This option is only effective when encryption.type is set to ipsec. | | encryption.nodeEncryption | bool | `false` | Enable encryption for pure node to node traffic. This option is only effective when encryption.type is set to "wireguard". | | encryption.secretName | string | `"cilium-ipsec-keys"` | Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. Name of the Kubernetes secret containing the encryption keys. This option is only effective when encryption.type is set to ipsec. | -| encryption.strictMode | object | `{"allowRemoteNodeIdentities":false,"cidr":"","enabled":false}` | Configure the WireGuard Pod2Pod strict mode. | -| encryption.strictMode.allowRemoteNodeIdentities | bool | `false` | Allow dynamic lookup of remote node identities. This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. | -| encryption.strictMode.cidr | string | `""` | CIDR for the WireGuard Pod2Pod strict mode. | -| encryption.strictMode.enabled | bool | `false` | Enable WireGuard Pod2Pod strict mode. | +| encryption.strictMode | object | `{"allowRemoteNodeIdentities":true,"enabled":false,"nodeCIDRList":[],"podCIDRList":[]}` | Configure the WireGuard strict mode. | +| encryption.strictMode.allowRemoteNodeIdentities | bool | `true` | Allow dynamic lookup of remote node identities. This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. This is also required when control-plane nodes are exempted from node-to-node encryption. | +| encryption.strictMode.enabled | bool | `false` | Enable WireGuard strict mode. | +| encryption.strictMode.nodeCIDRList | list | `[]` | nodeCIDRList for the WireGuard strict mode. | +| encryption.strictMode.podCIDRList | list | `[]` | podCIDRList for the WireGuard strict mode. | | encryption.type | string | `"ipsec"` | Encryption method. Can be either ipsec or wireguard. | | encryption.wireguard.persistentKeepalive | string | `"0s"` | Controls Wireguard PersistentKeepalive option. Set 0s to disable. | | encryption.wireguard.userspaceFallback | bool | `false` | Enables the fallback to the user-space implementation. | @@ -315,7 +325,7 @@ contributors across the globe, there is almost always someone available to help. | eni.subnetIDsFilter | list | `[]` | Filter via subnet IDs which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | | eni.subnetTagsFilter | list | `[]` | Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | | eni.updateEC2AdapterLimitViaAPI | bool | `true` | Update ENI Adapter limits from the EC2 API | -| envoy.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium-envoy"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-envoy. | +| envoy.affinity | object | `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"cilium.io/no-schedule","operator":"NotIn","values":["true"]}]}]}},"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]},"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium-envoy"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-envoy. | | envoy.annotations | object | `{}` | Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) | | envoy.connectTimeoutSeconds | int | `2` | Time in seconds after which a TCP connection attempt times out | | envoy.dnsPolicy | string | `nil` | DNS policy for Cilium envoy pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | @@ -328,7 +338,7 @@ contributors across the globe, there is almost always someone available to help. | envoy.extraVolumes | list | `[]` | Additional envoy volumes. | | envoy.healthPort | int | `9878` | TCP port for the health API. | | envoy.idleTimeoutDurationSeconds | int | `60` | Set Envoy upstream HTTP idle connection timeout seconds. Does not apply to connections with pending requests. Default 60s | -| envoy.image | object | `{"digest":"sha256:2b590be37624547d638a578a3f31278d3be53a1a2649ba888a9f15771628521e","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.27.2-ab187719b71b513150f30209569695adf16ec869","useDigest":true}` | Envoy container image. | +| envoy.image | object | `{"digest":"sha256:318eff387835ca2717baab42a84f35a83a5f9e7d519253df87269f80b9ff0171","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.33.4-1752151664-7c2edb0b44cf95f326d628b837fcdd845102ba68","useDigest":true}` | Envoy container image. | | envoy.livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | envoy.livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | | envoy.log.format | string | `"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"` | The format string to use for laying out the log message metadata of Envoy. | @@ -338,16 +348,18 @@ contributors across the globe, there is almost always someone available to help. | envoy.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for cilium-envoy. | | envoy.podAnnotations | object | `{}` | Annotations to be added to envoy pods | | envoy.podLabels | object | `{}` | Labels to be added to envoy pods | -| envoy.podSecurityContext | object | `{}` | Security Context for cilium-envoy pods. | +| envoy.podSecurityContext | object | `{"appArmorProfile":{"type":"Unconfined"}}` | Security Context for cilium-envoy pods. | +| envoy.podSecurityContext.appArmorProfile | object | `{"type":"Unconfined"}` | AppArmorProfile options for the `cilium-agent` and init containers | | envoy.priorityClassName | string | `nil` | The priority class to use for cilium-envoy. | +| envoy.prometheus | object | `{"enabled":true,"port":"9964","serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Configure Cilium Envoy Prometheus options. Note that some of these apply to either cilium-agent or cilium-envoy. | | envoy.prometheus.enabled | bool | `true` | Enable prometheus metrics for cilium-envoy | | envoy.prometheus.port | string | `"9964"` | Serve prometheus metrics for cilium-envoy on the configured port | | envoy.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-envoy | -| envoy.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| envoy.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) Note that this setting applies to both cilium-envoy _and_ cilium-agent with Envoy enabled. | | envoy.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | | envoy.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-envoy | -| envoy.prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-envoy | -| envoy.prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.metricRelabelings | string | `nil` | Metrics relabeling configs for the ServiceMonitor cilium-envoy or for cilium-agent with Envoy configured. | +| envoy.prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-envoy or for cilium-agent with Envoy configured. | | envoy.readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | | envoy.readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | | envoy.resources | object | `{}` | Envoy resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | @@ -360,6 +372,8 @@ contributors across the globe, there is almost always someone available to help. | envoy.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for cilium-envoy DaemonSet. | | envoy.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for envoy scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | envoy.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | cilium-envoy update strategy ref: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/#updating-a-daemonset | +| envoy.xffNumTrustedHopsL7PolicyEgress | int | `0` | Number of trusted hops regarding the x-forwarded-for and related HTTP headers for the egress L7 policy enforcement Envoy listeners. | +| envoy.xffNumTrustedHopsL7PolicyIngress | int | `0` | Number of trusted hops regarding the x-forwarded-for and related HTTP headers for the ingress L7 policy enforcement Envoy listeners. | | envoyConfig.enabled | bool | `false` | Enable CiliumEnvoyConfig CRD CiliumEnvoyConfig CRD can also be implicitly enabled by other options. | | envoyConfig.secretsNamespace | object | `{"create":true,"name":"cilium-secrets"}` | SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. | | envoyConfig.secretsNamespace.create | bool | `true` | Create secrets namespace for CiliumEnvoyConfig CRDs. | @@ -412,6 +426,14 @@ contributors across the globe, there is almost always someone available to help. | hostPort.enabled | bool | `false` | Enable hostPort service support. | | hubble.annotations | object | `{}` | Annotations to be added to all top-level hubble objects (resources under templates/hubble) | | hubble.enabled | bool | `true` | Enable Hubble (true by default). | +| hubble.export | object | `{"dynamic":{"config":{"configMapName":"cilium-flowlog-config","content":[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}],"createConfigMap":true},"enabled":false},"fileMaxBackups":5,"fileMaxSizeMb":10,"static":{"allowList":[],"denyList":[],"enabled":false,"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log"}}` | Hubble flows export. | +| hubble.export.dynamic | object | `{"config":{"configMapName":"cilium-flowlog-config","content":[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}],"createConfigMap":true},"enabled":false}` | - Dynamic exporters configuration. Dynamic exporters may be reconfigured without a need of agent restarts. | +| hubble.export.dynamic.config.configMapName | string | `"cilium-flowlog-config"` | -- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. | +| hubble.export.dynamic.config.content | list | `[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}]` | -- Exporters configuration in YAML format. | +| hubble.export.dynamic.config.createConfigMap | bool | `true` | -- True if helm installer should create config map. Switch to false if you want to self maintain the file content. | +| hubble.export.fileMaxBackups | int | `5` | - Defines max number of backup/rotated files. | +| hubble.export.fileMaxSizeMb | int | `10` | - Defines max file size of output file before it gets rotated. | +| hubble.export.static | object | `{"allowList":[],"denyList":[],"enabled":false,"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log"}` | - Static exporter configuration. Static exporter is bound to agent lifecycle. | | hubble.listenAddress | string | `":4244"` | An additional address for Hubble to listen to. Set this field ":4244" if you are enabling Hubble Relay, as it assumes that Hubble is listening on port 4244. | | hubble.metrics | object | `{"dashboards":{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null},"enableOpenMetrics":false,"enabled":null,"port":9965,"serviceAnnotations":{},"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":null,"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Hubble metrics configuration. See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics for more comprehensive documentation about Hubble metrics. | | hubble.metrics.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":null}` | Grafana dashboards for hubble grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | @@ -429,19 +451,22 @@ contributors across the globe, there is almost always someone available to help. | hubble.peerService.clusterDomain | string | `"cluster.local"` | The cluster domain to use to query the Hubble Peer service. It should be the local cluster. | | hubble.peerService.targetPort | int | `4244` | Target Port for the Peer service, must match the hubble.listenAddress' port. | | hubble.preferIpv6 | bool | `false` | Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. | -| hubble.redact | object | `{"enabled":false,"http":{"headers":{"allow":[],"deny":[]},"urlQuery":false},"kafka":{"apiKey":false}}` | Enables redacting sensitive information present in Layer 7 flows. | +| hubble.redact | object | `{"enabled":false,"http":{"headers":{"allow":[],"deny":[]},"urlQuery":false,"userInfo":true},"kafka":{"apiKey":true}}` | Enables redacting sensitive information present in Layer 7 flows. | | hubble.redact.http.headers.allow | list | `[]` | List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. Example: redact: enabled: true http: headers: allow: - traceparent - tracestate - Cache-Control You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" | | hubble.redact.http.headers.deny | list | `[]` | List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. Example: redact: enabled: true http: headers: deny: - Authorization - Proxy-Authorization You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" | | hubble.redact.http.urlQuery | bool | `false` | Enables redacting URL query (GET) parameters. Example: redact: enabled: true http: urlQuery: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.urlQuery="true" | -| hubble.redact.kafka.apiKey | bool | `false` | Enables redacting Kafka's API key. Example: redact: enabled: true kafka: apiKey: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.kafka.apiKey="true" | +| hubble.redact.http.userInfo | bool | `true` | Enables redacting user info, e.g., password when basic auth is used. Example: redact: enabled: true http: userInfo: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.userInfo="true" | +| hubble.redact.kafka.apiKey | bool | `true` | Enables redacting Kafka's API key. Example: redact: enabled: true kafka: apiKey: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.kafka.apiKey="true" | | hubble.relay.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for hubble-replay | | hubble.relay.annotations | object | `{}` | Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) | | hubble.relay.dialTimeout | string | `nil` | Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). | | hubble.relay.enabled | bool | `false` | Enable Hubble Relay (requires hubble.enabled=true) | | hubble.relay.extraEnv | list | `[]` | Additional hubble-relay environment variables. | +| hubble.relay.extraVolumeMounts | list | `[]` | Additional hubble-relay volumeMounts. | +| hubble.relay.extraVolumes | list | `[]` | Additional hubble-relay volumes. | | hubble.relay.gops.enabled | bool | `true` | Enable gops for hubble-relay | | hubble.relay.gops.port | int | `9893` | Configure gops listen port for hubble-relay | -| hubble.relay.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.15.0-pre.2","useDigest":false}` | Hubble-relay container image. | +| hubble.relay.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.15.19","useDigest":false}` | Hubble-relay container image. | | hubble.relay.listenHost | string | `""` | Host to listen to. Specify an empty string to bind to all the interfaces. | | hubble.relay.listenPort | string | `"4245"` | Port to listen to. | | hubble.relay.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | @@ -499,7 +524,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.backend.extraEnv | list | `[]` | Additional hubble-ui backend environment variables. | | hubble.ui.backend.extraVolumeMounts | list | `[]` | Additional hubble-ui backend volumeMounts. | | hubble.ui.backend.extraVolumes | list | `[]` | Additional hubble-ui backend volumes. | -| hubble.ui.backend.image | object | `{"digest":"sha256:1f86f3400827a0451e6332262467f894eeb7caf0eb8779bd951e2caa9d027cbe","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui-backend","tag":"v0.12.1","useDigest":true}` | Hubble-ui backend image. | +| hubble.ui.backend.image | object | `{"digest":"sha256:a034b7e98e6ea796ed26df8f4e71f83fc16465a19d166eff67a03b822c0bfa15","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui-backend","tag":"v0.13.2","useDigest":true}` | Hubble-ui backend image. | | hubble.ui.backend.livenessProbe.enabled | bool | `false` | Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) | | hubble.ui.backend.readinessProbe.enabled | bool | `false` | Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) | | hubble.ui.backend.resources | object | `{}` | Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. | @@ -509,7 +534,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.frontend.extraEnv | list | `[]` | Additional hubble-ui frontend environment variables. | | hubble.ui.frontend.extraVolumeMounts | list | `[]` | Additional hubble-ui frontend volumeMounts. | | hubble.ui.frontend.extraVolumes | list | `[]` | Additional hubble-ui frontend volumes. | -| hubble.ui.frontend.image | object | `{"digest":"sha256:9e5f81ee747866480ea1ac4630eb6975ff9227f9782b7c93919c081c33f38267","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui","tag":"v0.12.1","useDigest":true}` | Hubble-ui frontend image. | +| hubble.ui.frontend.image | object | `{"digest":"sha256:9e37c1296b802830834cc87342a9182ccbb71ffebb711971e849221bd9d59392","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-ui","tag":"v0.13.2","useDigest":true}` | Hubble-ui frontend image. | | hubble.ui.frontend.resources | object | `{}` | Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. | | hubble.ui.frontend.securityContext | object | `{}` | Hubble-ui frontend security context. | | hubble.ui.frontend.server.ipv6 | object | `{"enabled":true}` | Controls server listener for ipv6 | @@ -536,7 +561,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | hubble-ui update strategy. | | identityAllocationMode | string | `"crd"` | Method to use for identity allocation (`crd` or `kvstore`). | | identityChangeGracePeriod | string | `"5s"` | Time to wait before using new identity on endpoint identity change. | -| image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.15.0-pre.2","useDigest":false}` | Agent container image. | +| image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.15.19","useDigest":false}` | Agent container image. | | imagePullSecrets | string | `nil` | Configure image pull secrets for pulling container images | | ingressController.default | bool | `false` | Set cilium ingress controller to be the default ingress controller This will let cilium ingress controller route entries without ingress class set | | ingressController.defaultSecretName | string | `nil` | Default secret name for ingresses without .spec.tls[].secretName set. | @@ -544,7 +569,7 @@ contributors across the globe, there is almost always someone available to help. | ingressController.enableProxyProtocol | bool | `false` | Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. | | ingressController.enabled | bool | `false` | Enable cilium ingress controller This will automatically set enable-envoy-config as well. | | ingressController.enforceHttps | bool | `true` | Enforce https for host having matching TLS host in Ingress. Incoming traffic to http listener will return 308 http error code with respective location in header. | -| ingressController.ingressLBAnnotationPrefixes | list | `["service.beta.kubernetes.io","service.kubernetes.io","cloud.google.com"]` | IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate from Ingress to the Load Balancer service | +| ingressController.ingressLBAnnotationPrefixes | list | `["service.beta.kubernetes.io","service.kubernetes.io","cloud.google.com"]` | IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service | | ingressController.loadbalancerMode | string | `"dedicated"` | Default ingress load balancer mode Supported values: shared, dedicated For granular control, use the following annotations on the ingress resource ingress.cilium.io/loadbalancer-mode: shared|dedicated, | | ingressController.secretsNamespace | object | `{"create":true,"name":"cilium-secrets","sync":true}` | SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. | | ingressController.secretsNamespace.create | bool | `true` | Create secrets namespace for Ingress. | @@ -560,6 +585,7 @@ contributors across the globe, there is almost always someone available to help. | ingressController.service.name | string | `"cilium-ingress"` | Service name | | ingressController.service.secureNodePort | string | `nil` | Configure a specific nodePort for secure HTTPS traffic on the shared LB service | | ingressController.service.type | string | `"LoadBalancer"` | Service type for the shared LB service | +| initResources | object | `{}` | resources & limits for the agent init containers | | installNoConntrackIptablesRules | bool | `false` | Install Iptables rules to skip netfilter connection tracking on all pod traffic. This option is only effective when Cilium is running in direct routing and full KPR mode. Moreover, this option cannot be enabled when Cilium is running in a managed Kubernetes environment or in a chained CNI setup. | | ipMasqAgent | object | `{"enabled":false}` | Configure the eBPF-based ip-masq-agent | | ipam.ciliumNodeUpdateRate | string | `"15s"` | Maximum rate at which the CiliumNode custom resource is updated. | @@ -576,9 +602,9 @@ contributors across the globe, there is almost always someone available to help. | ipv6.enabled | bool | `false` | Enable IPv6 support. | | ipv6NativeRoutingCIDR | string | `""` | Allows to explicitly specify the IPv6 CIDR for native routing. When specified, Cilium assumes networking for this CIDR is preconfigured and hands traffic destined for that range to the Linux network stack without applying any SNAT. Generally speaking, specifying a native routing CIDR implies that Cilium can depend on the underlying networking stack to route packets to their destination. To offer a concrete example, if Cilium is configured to use direct routing and the Kubernetes CIDR is included in the native routing CIDR, the user must configure the routes to reach pods, either manually or by setting the auto-direct-node-routes flag. | | k8s | object | `{}` | Configure Kubernetes specific configuration | -| k8sClientRateLimit | object | `{"burst":10,"qps":5}` | Configure the client side rate limit for the agent and operator If the amount of requests to the Kubernetes API server exceeds the configured rate limit, the agent and operator will start to throttle requests by delaying them until there is budget or the request times out. | -| k8sClientRateLimit.burst | int | `10` | The burst request rate in requests per second. The rate limiter will allow short bursts with a higher rate. | -| k8sClientRateLimit.qps | int | `5` | The sustained request rate in requests per second. | +| k8sClientRateLimit | object | `{"burst":null,"qps":null}` | Configure the client side rate limit for the agent and operator If the amount of requests to the Kubernetes API server exceeds the configured rate limit, the agent and operator will start to throttle requests by delaying them until there is budget or the request times out. | +| k8sClientRateLimit.burst | int | 10 for k8s up to 1.26. 20 for k8s version 1.27+ | The burst request rate in requests per second. The rate limiter will allow short bursts with a higher rate. | +| k8sClientRateLimit.qps | int | 5 for k8s up to 1.26. 10 for k8s version 1.27+ | The sustained request rate in requests per second. | | k8sNetworkPolicy.enabled | bool | `true` | Enable support for K8s NetworkPolicy | | k8sServiceHost | string | `""` | Kubernetes service host | | k8sServicePort | string | `""` | Kubernetes service port | @@ -596,7 +622,8 @@ contributors across the globe, there is almost always someone available to help. | l7Proxy | bool | `true` | Enable Layer 7 network policy. | | livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | -| loadBalancer | object | `{"l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | +| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | +| loadBalancer.acceleration | string | `"disabled"` | acceleration is the option to accelerate service handling via XDP Applicable values can be: disabled (do not use XDP), native (XDP BPF program is run directly out of the networking driver's early receive path), or best-effort (use native mode XDP acceleration on devices that support it). | | loadBalancer.l7 | object | `{"algorithm":"round_robin","backend":"disabled","ports":[]}` | L7 LoadBalancer | | loadBalancer.l7.algorithm | string | `"round_robin"` | Default LB algorithm The default LB algorithm to be used for services, which can be overridden by the service annotation (e.g. service.cilium.io/lb-l7-algorithm) Applicable values: round_robin, least_request, random | | loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing by way of service annotation. | @@ -623,10 +650,12 @@ contributors across the globe, there is almost always someone available to help. | nodeinit.extraEnv | list | `[]` | Additional nodeinit environment variables. | | nodeinit.extraVolumeMounts | list | `[]` | Additional nodeinit volumeMounts. | | nodeinit.extraVolumes | list | `[]` | Additional nodeinit volumes. | -| nodeinit.image | object | `{"override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/startup-script","tag":"62093c5c233ea914bfa26a10ba41f8780d9b737f"}` | node-init image. | +| nodeinit.image | object | `{"digest":"sha256:8d7b41c4ca45860254b3c19e20210462ef89479bb6331d6760c4e609d651b29c","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/startup-script","tag":"c54c7edeab7fde4da68e59acd319ab24af242c3f","useDigest":true}` | node-init image. | | nodeinit.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for nodeinit pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | nodeinit.podAnnotations | object | `{}` | Annotations to be added to node-init pods. | | nodeinit.podLabels | object | `{}` | Labels to be added to node-init pods. | +| nodeinit.podSecurityContext | object | `{"appArmorProfile":{"type":"Unconfined"}}` | Security Context for cilium-node-init pods. | +| nodeinit.podSecurityContext.appArmorProfile | object | `{"type":"Unconfined"}` | AppArmorProfile options for the `cilium-node-init` and init containers | | nodeinit.prestop | object | `{"postScript":"","preScript":""}` | prestop offers way to customize prestop nodeinit script (pre and post position) | | nodeinit.priorityClassName | string | `""` | The priority class to use for the nodeinit pod. | | nodeinit.resources | object | `{"requests":{"cpu":"100m","memory":"100Mi"}}` | nodeinit resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | @@ -647,7 +676,7 @@ contributors across the globe, there is almost always someone available to help. | operator.extraVolumes | list | `[]` | Additional cilium-operator volumes. | | operator.identityGCInterval | string | `"15m0s"` | Interval for identity garbage collection. | | operator.identityHeartbeatTimeout | string | `"30m0s"` | Timeout for identity heartbeats. | -| operator.image | object | `{"alibabacloudDigest":"","awsDigest":"","azureDigest":"","genericDigest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.15.0-pre.2","useDigest":false}` | cilium-operator image. | +| operator.image | object | `{"alibabacloudDigest":"","awsDigest":"","azureDigest":"","genericDigest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.15.19","useDigest":false}` | cilium-operator image. | | operator.nodeGCInterval | string | `"5m0s"` | Interval for cilium node garbage collection. | | operator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | operator.podAnnotations | object | `{}` | Annotations to be added to cilium-operator pods | @@ -685,7 +714,8 @@ contributors across the globe, there is almost always someone available to help. | pmtuDiscovery.enabled | bool | `false` | Enable path MTU discovery to send ICMP fragmentation-needed replies to the client. | | podAnnotations | object | `{}` | Annotations to be added to agent pods | | podLabels | object | `{}` | Labels to be added to agent pods | -| podSecurityContext | object | `{}` | Security Context for cilium-agent pods. | +| podSecurityContext | object | `{"appArmorProfile":{"type":"Unconfined"}}` | Security Context for cilium-agent pods. | +| podSecurityContext.appArmorProfile | object | `{"type":"Unconfined"}` | AppArmorProfile options for the `cilium-agent` and init containers | | policyCIDRMatchMode | string | `nil` | policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. The possible value is "nodes". | | policyEnforcementMode | string | `"default"` | The agent can be put into one of the three policy enforcement modes: default, always and never. ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes | | pprof.address | string | `"localhost"` | Configure pprof listen address for cilium-agent | @@ -697,7 +727,7 @@ contributors across the globe, there is almost always someone available to help. | preflight.extraEnv | list | `[]` | Additional preflight environment variables. | | preflight.extraVolumeMounts | list | `[]` | Additional preflight volumeMounts. | | preflight.extraVolumes | list | `[]` | Additional preflight volumes. | -| preflight.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.15.0-pre.2","useDigest":false}` | Cilium pre-flight image. | +| preflight.image | object | `{"digest":"","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.15.19","useDigest":false}` | Cilium pre-flight image. | | preflight.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for preflight pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | preflight.podAnnotations | object | `{}` | Annotations to be added to preflight pods | | preflight.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -732,7 +762,7 @@ contributors across the globe, there is almost always someone available to help. | rbac.create | bool | `true` | Enable creation of Resource-Based Access Control configuration. | | readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | | readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | -| remoteNodeIdentity | bool | `true` | Enable use of the remote node identity. ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity | +| remoteNodeIdentity | bool | `true` | Enable use of the remote node identity. ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity Deprecated without replacement in 1.15. To be removed in 1.16. | | resourceQuotas | object | `{"cilium":{"hard":{"pods":"10k"}},"enabled":false,"operator":{"hard":{"pods":"15"}}}` | Enable resource quotas for priority classes used in the cluster. | | resources | object | `{}` | Agent resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | | rollOutCiliumPods | bool | `false` | Roll out cilium agent pods automatically when configmap is updated. | @@ -749,6 +779,7 @@ contributors across the globe, there is almost always someone available to help. | serviceAccounts.clustermeshcertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"clustermesh-apiserver-generate-certs"}` | Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob | | serviceAccounts.hubblecertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"hubble-generate-certs"}` | Hubblecertgen is used if hubble.tls.auto.method=cronJob | | serviceAccounts.nodeinit.enabled | bool | `false` | Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by this issue. Name and automount can be configured, if enabled is set to true. Otherwise, they are ignored. Enabled can be removed once the issue is fixed. Cilium-nodeinit DS must also be fixed. | +| serviceNoBackendResponse | string | `"reject"` | Configure what the response should be to traffic for a service without backends. "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". Possible values: - reject (default) - drop | | sleepAfterInit | bool | `false` | Do not run Cilium agent when running with clean mode. Useful to completely uninstall Cilium as it will stop Cilium from starting and create artifacts in the node. | | socketLB | object | `{"enabled":false}` | Configure socket LB | | socketLB.enabled | bool | `false` | Enable socket LB | @@ -756,6 +787,8 @@ contributors across the globe, there is almost always someone available to help. | startupProbe.periodSeconds | int | `2` | interval between checks of the startup probe | | svcSourceRangeCheck | bool | `true` | Enable check of service source ranges (currently, only for LoadBalancer). | | synchronizeK8sNodes | bool | `true` | Synchronize Kubernetes nodes to kvstore and perform CNP GC. | +| sysctlfix | object | `{"enabled":true}` | Configure sysctl override described in #20072. | +| sysctlfix.enabled | bool | `true` | Enable the sysctl override. When enabled, the init container will mount the /proc of the host so that the `sysctlfix` utility can execute. | | terminationGracePeriodSeconds | int | `1` | Configure termination grace period for cilium-agent DaemonSet. | | tls | object | `{"ca":{"cert":"","certValidityDuration":1095,"key":""},"caBundle":{"enabled":false,"key":"ca.crt","name":"cilium-root-ca.crt","useSecret":false},"secretsBackend":"local"}` | Configure TLS configuration in the agent. | | tls.ca | object | `{"cert":"","certValidityDuration":1095,"key":""}` | Base64 encoded PEM values for the CA certificate and private key. This can be used as common CA to generate certificates used by hubble and clustermesh components. It is neither required nor used when cert-manager is used to generate the certificates. | @@ -769,7 +802,6 @@ contributors across the globe, there is almost always someone available to help. | tls.caBundle.useSecret | bool | `false` | Use a Secret instead of a ConfigMap. | | tls.secretsBackend | string | `"local"` | This configures how the Cilium agent loads the secrets used TLS-aware CiliumNetworkPolicies (namely the secrets referenced by terminatingTLS and originatingTLS). Possible values: - local - k8s | | tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for agent scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | -| tunnel | string | `"vxlan"` | Configure the encapsulation configuration for communication between nodes. Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. Possible values: - disabled - vxlan - geneve | | tunnelPort | int | Port 8472 for VXLAN, Port 6081 for Geneve | Configure VXLAN and Geneve tunnel port. | | tunnelProtocol | string | `"vxlan"` | Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. Possible values: - "" - vxlan - geneve | | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | Cilium agent update strategy | diff --git a/internal/constellation/helm/charts/cilium/README.md.gotmpl b/internal/constellation/helm/charts/cilium/README.md.gotmpl index db2d81b74..4aa7da8f9 100644 --- a/internal/constellation/helm/charts/cilium/README.md.gotmpl +++ b/internal/constellation/helm/charts/cilium/README.md.gotmpl @@ -48,7 +48,7 @@ offer from the [Getting Started Guides page](https://docs.cilium.io/en/stable/ge ## Getting Help The best way to get help if you get stuck is to ask a question on the -[Cilium Slack channel](https://cilium.herokuapp.com/). With Cilium +[Cilium Slack channel](https://slack.cilium.io). With Cilium contributors across the globe, there is almost always someone available to help. {{ template "chart.valuesSection" . }} diff --git a/internal/constellation/helm/charts/cilium/files/agent/poststart-eni.bash b/internal/constellation/helm/charts/cilium/files/agent/poststart-eni.bash index 66fccf457..a57d89682 100644 --- a/internal/constellation/helm/charts/cilium/files/agent/poststart-eni.bash +++ b/internal/constellation/helm/charts/cilium/files/agent/poststart-eni.bash @@ -11,9 +11,9 @@ set -o nounset # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). -if [[ "$(iptables-save | grep -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; +if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore fi echo 'Done!' diff --git a/internal/constellation/helm/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json b/internal/constellation/helm/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json index 992c8524a..94af2eac3 100644 --- a/internal/constellation/helm/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json +++ b/internal/constellation/helm/charts/cilium/files/cilium-agent/dashboards/cilium-dashboard.json @@ -5823,7 +5823,7 @@ "refId": "C" }, { - "expr": "sum(cilium_policy_change_total{k8s_app=\"cilium\", pod=~\"$pod\"}, outcome=\"fail\") by (pod)", + "expr": "sum(cilium_policy_change_total{k8s_app=\"cilium\", pod=~\"$pod\", outcome=\"fail\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "policy change errors", diff --git a/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json b/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json index 3d8656c31..87939f699 100644 --- a/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json +++ b/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.json @@ -36,7 +36,7 @@ "prefix": "/metrics" }, "route": { - "cluster": "envoy-admin", + "cluster": "/envoy-admin", "prefix_rewrite": "/stats/prometheus" } } @@ -102,7 +102,7 @@ "prefix": "/healthz" }, "route": { - "cluster": "envoy-admin", + "cluster": "/envoy-admin", "prefix_rewrite": "/ready" } } @@ -245,11 +245,11 @@ } }, { - "name": "envoy-admin", + "name": "/envoy-admin", "type": "STATIC", "connectTimeout": "{{ .Values.envoy.connectTimeoutSeconds }}s", "loadAssignment": { - "clusterName": "envoy-admin", + "clusterName": "/envoy-admin", "endpoints": [ { "lbEndpoints": [ @@ -301,6 +301,14 @@ "resourceApiVersion": "V3" } }, + "bootstrapExtensions": [ + { + "name": "envoy.bootstrap.internal_listener", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + } + } + ], "layeredRuntime": { "layers": [ { diff --git a/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml b/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml new file mode 100644 index 000000000..920837268 --- /dev/null +++ b/internal/constellation/helm/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml @@ -0,0 +1,232 @@ +node: + id: "host~127.0.0.1~no-id~localdomain" + cluster: "ingress-cluster" +staticResources: + listeners: + {{- if .Values.envoy.prometheus.enabled }} + - name: "envoy-prometheus-metrics-listener" + address: + socketAddress: + address: {{ .Values.ipv4.enabled | ternary "0.0.0.0" "::" | quote }} + portValue: {{ .Values.envoy.prometheus.port }} + {{- if and .Values.ipv4.enabled .Values.ipv6.enabled }} + additionalAddresses: + - address: + socketAddress: + address: "::" + portValue: {{ .Values.envoy.prometheus.port }} + {{- end }} + filterChains: + - filters: + - name: "envoy.filters.network.http_connection_manager" + typedConfig: + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + statPrefix: "envoy-prometheus-metrics-listener" + routeConfig: + virtualHosts: + - name: "prometheus_metrics_route" + domains: + - "*" + routes: + - name: "prometheus_metrics_route" + match: + prefix: "/metrics" + route: + cluster: "/envoy-admin" + prefixRewrite: "/stats/prometheus" + httpFilters: + - name: "envoy.filters.http.router" + typedConfig: + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + internalAddressConfig: + cidrRanges: + {{- if .Values.ipv4.enabled }} + - addressPrefix: "10.0.0.0" + prefixLen: 8 + - addressPrefix: "172.16.0.0" + prefixLen: 12 + - addressPrefix: "192.168.0.0" + prefixLen: 16 + - addressPrefix: "127.0.0.1" + prefixLen: 32 + {{- end }} + {{- if .Values.ipv6.enabled }} + - addressPrefix: "::1" + prefixLen: 128 + {{- end }} + streamIdleTimeout: "0s" + {{- end }} + - name: "envoy-health-listener" + address: + socketAddress: + address: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} + portValue: {{ .Values.envoy.healthPort }} + {{- if and .Values.ipv4.enabled .Values.ipv6.enabled }} + additionalAddresses: + - address: + socketAddress: + address: "::1" + portValue: {{ .Values.envoy.healthPort }} + {{- end }} + filterChains: + - filters: + - name: "envoy.filters.network.http_connection_manager" + typedConfig: + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + statPrefix: "envoy-health-listener" + routeConfig: + virtual_hosts: + - name: "health" + domains: + - "*" + routes: + - name: "health" + match: + prefix: "/healthz" + route: + cluster: "/envoy-admin" + prefixRewrite: "/ready" + httpFilters: + - name: "envoy.filters.http.router" + typedConfig: + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + internalAddressConfig: + cidrRanges: + {{- if .Values.ipv4.enabled }} + - addressPrefix: "10.0.0.0" + prefixLen: 8 + - addressPrefix: "172.16.0.0" + prefixLen: 12 + - addressPrefix: "192.168.0.0" + prefixLen: 16 + - addressPrefix: "127.0.0.1" + prefixLen: 32 + {{- end }} + {{- if .Values.ipv6.enabled }} + - addressPrefix: "::1" + prefixLen: 128 + {{- end }} + streamIdleTimeout: "0s" + clusters: + - name: "ingress-cluster" + type: "ORIGINAL_DST" + connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s" + lbPolicy: "CLUSTER_PROVIDED" + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + commonHttpProtocolOptions: + idleTimeout: "{{ .Values.envoy.idleTimeoutDurationSeconds }}s" + maxConnectionDuration: "{{ .Values.envoy.maxConnectionDurationSeconds }}s" + maxRequestsPerConnection: {{ .Values.envoy.maxRequestsPerConnection }} + useDownstreamProtocolConfig: {} + cleanupInterval: "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + - name: "egress-cluster-tls" + type: "ORIGINAL_DST" + connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s" + lbPolicy: "CLUSTER_PROVIDED" + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + commonHttpProtocolOptions: + idleTimeout: "{{ .Values.envoy.idleTimeoutDurationSeconds }}s" + maxConnectionDuration: "{{ .Values.envoy.maxConnectionDurationSeconds }}s" + maxRequestsPerConnection: {{ .Values.envoy.maxRequestsPerConnection }} + upstreamHttpProtocolOptions: {} + useDownstreamProtocolConfig: {} + cleanupInterval: "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + transportSocket: + name: "cilium.tls_wrapper" + typedConfig: + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + - name: "egress-cluster" + type: "ORIGINAL_DST" + connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s" + lbPolicy: "CLUSTER_PROVIDED" + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + commonHttpProtocolOptions: + idleTimeout: "{{ .Values.envoy.idleTimeoutDurationSeconds }}s" + maxConnectionDuration: "{{ .Values.envoy.maxConnectionDurationSeconds }}s" + maxRequestsPerConnection: {{ .Values.envoy.maxRequestsPerConnection }} + useDownstreamProtocolConfig: {} + cleanupInterval: "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + - name: "ingress-cluster-tls" + type: "ORIGINAL_DST" + connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s" + lbPolicy: "CLUSTER_PROVIDED" + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + commonHttpProtocolOptions: + idleTimeout: "{{ .Values.envoy.idleTimeoutDurationSeconds }}s" + maxConnectionDuration: "{{ .Values.envoy.maxConnectionDurationSeconds }}s" + maxRequestsPerConnection: {{ .Values.envoy.maxRequestsPerConnection }} + upstreamHttpProtocolOptions: {} + useDownstreamProtocolConfig: {} + cleanupInterval: "{{ .Values.envoy.connectTimeoutSeconds }}.500s" + transportSocket: + name: "cilium.tls_wrapper" + typedConfig: + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + - name: "xds-grpc-cilium" + type: "STATIC" + connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s" + loadAssignment: + clusterName: "xds-grpc-cilium" + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: "/var/run/cilium/envoy/sockets/xds.sock" + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicitHttpConfig: + http2ProtocolOptions: {} + - name: "/envoy-admin" + type: "STATIC" + connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s" + loadAssignment: + clusterName: "/envoy-admin" + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: "/var/run/cilium/envoy/sockets/admin.sock" +dynamicResources: + ldsConfig: + apiConfigSource: + apiType: "GRPC" + transportApiVersion: "V3" + grpcServices: + - envoyGrpc: + clusterName: "xds-grpc-cilium" + setNodeOnFirstMessageOnly: true + resourceApiVersion: "V3" + cdsConfig: + apiConfigSource: + apiType: "GRPC" + transportApiVersion: "V3" + grpcServices: + - envoyGrpc: + clusterName: "xds-grpc-cilium" + setNodeOnFirstMessageOnly: true + resourceApiVersion: "V3" +bootstrapExtensions: +- name: "envoy.bootstrap.internal_listener" + typedConfig: + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" +overloadManager: + resourceMonitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typedConfig: + "@type": "type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig" + max_active_downstream_connections: "50000" +admin: + address: + pipe: + path: "/var/run/cilium/envoy/sockets/admin.sock" diff --git a/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dashboard.json b/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dashboard.json index 12de657d2..0ff1dcbec 100644 --- a/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dashboard.json +++ b/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dashboard.json @@ -3194,7 +3194,23 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] }, "time": { "from": "now-6h", @@ -3226,7 +3242,7 @@ ] }, "timezone": "", - "title": "Hubble", + "title": "Hubble Metrics and Monitoring", "uid": "5HftnJAWz", "version": 24 } diff --git a/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json b/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json index d286fdb3a..57f804cf2 100644 --- a/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json +++ b/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-dns-namespace.json @@ -484,7 +484,7 @@ "includeAll": false, "label": "Data Source", "multi": false, - "name": "prometheus_datasource", + "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", diff --git a/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json b/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json index d0cf9d3b4..cddb473d7 100644 --- a/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json +++ b/internal/constellation/helm/charts/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json @@ -883,7 +883,7 @@ "includeAll": false, "label": "Data Source", "multi": false, - "name": "prometheus_datasource", + "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "queryValue": "", diff --git a/internal/constellation/helm/charts/cilium/files/nodeinit/startup.bash b/internal/constellation/helm/charts/cilium/files/nodeinit/startup.bash index 975b71fa7..aaaba0925 100644 --- a/internal/constellation/helm/charts/cilium/files/nodeinit/startup.bash +++ b/internal/constellation/helm/charts/cilium/files/nodeinit/startup.bash @@ -100,7 +100,7 @@ then # Since that version containerd no longer allows missing configuration for the CNI, # not even for pods with hostNetwork set to true. Thus, we add a temporary one. # This will be replaced with the real config by the agent pod. - echo -e "{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}" > /etc/cni/net.d/05-cilium.conf + echo -e '{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}' > /etc/cni/net.d/05-cilium.conf fi # Start containerd. It won't create it's CNI configuration file anymore. diff --git a/internal/constellation/helm/charts/cilium/templates/_helpers.tpl b/internal/constellation/helm/charts/cilium/templates/_helpers.tpl index 3e5429e2a..39b3d6955 100644 --- a/internal/constellation/helm/charts/cilium/templates/_helpers.tpl +++ b/internal/constellation/helm/charts/cilium/templates/_helpers.tpl @@ -43,62 +43,7 @@ where: {{- if $priorityClass }} {{- $priorityClass }} {{- else if and $root.Values.enableCriticalPriorityClass $criticalPriorityClass -}} - {{- if and (eq $root.Release.Namespace "kube-system") (semverCompare ">=1.10-0" $root.Capabilities.KubeVersion.Version) -}} - {{- $criticalPriorityClass }} - {{- else if semverCompare ">=1.17-0" $root.Capabilities.KubeVersion.Version -}} - {{- $criticalPriorityClass }} - {{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for ingress. -*/}} -{{- define "ingress.apiVersion" -}} -{{- if semverCompare ">=1.16-0, <1.19-0" .Capabilities.KubeVersion.Version -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- else if semverCompare "^1.19-0" .Capabilities.KubeVersion.Version -}} -{{- print "networking.k8s.io/v1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate backend for Hubble UI ingress. -*/}} -{{- define "ingress.paths" -}} -{{ if semverCompare ">=1.4-0, <1.19-0" .Capabilities.KubeVersion.Version -}} -backend: - serviceName: hubble-ui - servicePort: http -{{- else if semverCompare "^1.19-0" .Capabilities.KubeVersion.Version -}} -pathType: Prefix -backend: - service: - name: hubble-ui - port: - name: http -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for cronjob. -*/}} -{{- define "cronjob.apiVersion" -}} -{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version -}} -{{- print "batch/v1" -}} -{{- else -}} -{{- print "batch/v1beta1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for podDisruptionBudget. -*/}} -{{- define "podDisruptionBudget.apiVersion" -}} -{{- if semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version -}} -{{- print "policy/v1" -}} -{{- else -}} -{{- print "policy/v1beta1" -}} + {{- $criticalPriorityClass }} {{- end -}} {{- end -}} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-agent/daemonset.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-agent/daemonset.yaml index 146a32374..3e288525f 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-agent/daemonset.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-agent/daemonset.yaml @@ -53,6 +53,7 @@ spec: cilium.io/cilium-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-configmap.yaml") . | sha256sum | quote }} {{- end }} {{- if not .Values.securityContext.privileged }} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} # Set app AppArmor's profile to "unconfined". The value of this annotation # can be modified as long users know which profiles they have available # in AppArmor. @@ -63,6 +64,7 @@ spec: container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" {{- end }} {{- end }} + {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -81,6 +83,11 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- /* K8s version lower than 1.30.0 don't support the "appArmorProfile" field, */}} + {{- /* thus we have to remove it. */}} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} + {{- $_ := unset .Values.podSecurityContext "appArmorProfile" }} + {{- end }} {{- with .Values.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} @@ -115,7 +122,6 @@ spec: {{- with .Values.extraArgs }} {{- toYaml . | trim | nindent 8 }} {{- end }} - {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} startupProbe: httpGet: host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} @@ -128,7 +134,7 @@ spec: failureThreshold: {{ .Values.startupProbe.failureThreshold }} periodSeconds: {{ .Values.startupProbe.periodSeconds }} successThreshold: 1 - {{- end }} + initialDelaySeconds: 5 livenessProbe: {{- if or .Values.keepDeprecatedProbes $defaultKeepDeprecatedProbes }} exec: @@ -146,14 +152,6 @@ spec: - name: "brief" value: "true" {{- end }} - {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} - # The initial delay for the liveness probe is intentionally large to - # avoid an endless kill & restart cycle if in the event that the initial - # bootstrapping takes longer than expected. - # Starting from Kubernetes 1.20, we are using startupProbe instead - # of this field. - initialDelaySeconds: 120 - {{- end }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} successThreshold: 1 failureThreshold: {{ .Values.livenessProbe.failureThreshold }} @@ -175,9 +173,6 @@ spec: - name: "brief" value: "true" {{- end }} - {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} - initialDelaySeconds: 5 - {{- end }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} successThreshold: 1 failureThreshold: {{ .Values.readinessProbe.failureThreshold }} @@ -200,6 +195,7 @@ spec: valueFrom: resourceFieldRef: resource: limits.memory + divisor: '1' {{- if .Values.k8sServiceHost }} - name: KUBERNETES_SERVICE_HOST value: {{ .Values.k8sServiceHost | quote }} @@ -375,6 +371,11 @@ spec: mountPropagation: {{ .mountPropagation }} {{- end }} {{- end }} + {{- if .Values.hubble.export.dynamic.enabled }} + - name: hubble-flowlog-config + mountPath: /flowlog-config + readOnly: true + {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -399,6 +400,9 @@ spec: volumeMounts: - name: cilium-run mountPath: /var/run/cilium + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.monitor.resources }} resources: {{- toYaml . | trim | nindent 10 }} @@ -423,6 +427,9 @@ spec: {{- if (not (kindIs "invalid" .Values.daemon.blockedConfigOverrides)) }} - "--deny-config-keys={{.Values.daemon.blockedConfigOverrides}}" {{- end }} + {{- if .Values.kubeConfigPath }} + - "--k8s-kubeconfig-path={{ .Values.kubeConfigPath }}" + {{- end }} env: - name: K8S_NODE_NAME valueFrom: @@ -448,6 +455,14 @@ spec: volumeMounts: - name: tmp mountPath: /tmp + {{- if .Values.kubeConfigPath }} + - name: kube-config + mountPath: {{ .Values.kubeConfigPath }} + readOnly: true + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} terminationMessagePolicy: FallbackToLogsOnError {{- if .Values.cgroup.autoMount.enabled }} # Required to mount cgroup2 filesystem on the underlying Kubernetes node. @@ -498,9 +513,15 @@ spec: drop: - ALL {{- end}} + {{- end }} + {{- if .Values.sysctlfix.enabled }} - name: apply-sysctl-overwrites image: {{ include "cilium.image" .Values.image | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} env: - name: BIN_PATH value: {{ .Values.cni.binPath }} @@ -546,6 +567,10 @@ spec: - name: mount-bpf-fs image: {{ include "cilium.image" .Values.image | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} args: - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' command: @@ -567,6 +592,10 @@ spec: - name: wait-for-node-init image: {{ include "cilium.image" .Values.image | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} command: - sh - -c @@ -644,14 +673,21 @@ spec: mountPropagation: HostToContainer - name: cilium-run mountPath: /var/run/cilium - {{- with .Values.nodeinit.resources }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.initResources }} resources: {{- toYaml . | trim | nindent 10 }} {{- end }} - {{- if and .Values.waitForKubeProxy (and (ne $kubeProxyReplacement "strict") (ne $kubeProxyReplacement "true")) }} + {{- if and .Values.waitForKubeProxy (and (ne (toString $kubeProxyReplacement) "strict") (ne (toString $kubeProxyReplacement) "true")) }} - name: wait-for-kube-proxy image: {{ include "cilium.image" .Values.image | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.initResources }} + resources: + {{- toYaml . | trim | nindent 10 }} + {{- end }} securityContext: privileged: true command: @@ -688,10 +724,10 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} command: - "/install-plugin.sh" + {{- with .Values.cni.resources }} resources: - requests: - cpu: 100m - memory: 10Mi + {{- toYaml . | trim | nindent 10 }} + {{- end }} securityContext: {{- if .Values.securityContext.privileged }} privileged: true @@ -709,9 +745,40 @@ spec: - name: cni-path mountPath: /host/opt/cni/bin {{- end }} # .Values.cni.install + - name: firewall-pods + image: {{ include "cilium.image" .Values.image | quote }} + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -exc + - | + pref=32 + for interface in $(ip route | awk '/^default/ { print $5 }'); do + tc qdisc add dev "${interface}" clsact || true + tc filter del dev "${interface}" ingress pref "${pref}" 2>/dev/null || true + handle=0 + for cidr in ${POD_CIDRS}; do + handle=$((handle + 1)) + tc filter replace dev "${interface}" ingress pref "${pref}" handle "${handle}" protocol ip flower dst_ip "${cidr}" action drop + done + done + env: + - name: POD_CIDRS + valueFrom: + configMapKeyRef: + key: encryption-strict-mode-pod-cidrs + name: cilium-config + optional: true + resources: + requests: + cpu: 100m + memory: 20Mi + securityContext: + capabilities: + add: + - NET_ADMIN restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.priorityClassName "system-node-critical") }} - serviceAccount: {{ .Values.serviceAccounts.cilium.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.cilium.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.cilium.automount }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} @@ -761,8 +828,8 @@ spec: path: /sys/fs/bpf type: DirectoryOrCreate {{- end }} - {{- if .Values.cgroup.autoMount.enabled }} - # To mount cgroup2 filesystem on the host + {{- if or .Values.cgroup.autoMount.enabled .Values.sysctlfix.enabled }} + # To mount cgroup2 filesystem on the host or apply sysctlfix - name: hostproc hostPath: path: /proc @@ -929,6 +996,12 @@ spec: path: client-ca.crt {{- end }} {{- end }} + {{- if .Values.hubble.export.dynamic.enabled }} + - name: hubble-flowlog-config + configMap: + name: {{ .Values.hubble.export.dynamic.config.configMapName }} + optional: true + {{- end }} {{- range .Values.extraHostPathMounts }} - name: {{ .name }} hostPath: diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-agent/serviceaccount.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-agent/serviceaccount.yaml index acef1f1fc..2c2cf3992 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-agent/serviceaccount.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-agent/serviceaccount.yaml @@ -4,10 +4,6 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.cilium.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.cilium.annotations }} - annotations: - {{- toYaml .Values.serviceAccounts.cilium.annotations | nindent 4 }} - {{- end }} {{- if or .Values.serviceAccounts.cilium.annotations .Values.annotations }} annotations: {{- with .Values.annotations }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-agent/servicemonitor.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-agent/servicemonitor.yaml index d7c5e5e3c..c2ffa66c2 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-agent/servicemonitor.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-agent/servicemonitor.yaml @@ -39,6 +39,20 @@ spec: metricRelabelings: {{- toYaml . | nindent 4 }} {{- end }} + {{- if .Values.envoy.prometheus.serviceMonitor.enabled }} + - port: envoy-metrics + interval: {{ .Values.envoy.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.envoy.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.envoy.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} targetLabels: - k8s-app {{- if .Values.prometheus.serviceMonitor.jobLabel }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-configmap.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-configmap.yaml index 3c1a0de06..9d393c311 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-configmap.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-configmap.yaml @@ -1,6 +1,5 @@ {{- if and (.Values.agent) (not .Values.preflight.enabled) }} {{- /* Default values with backwards compatibility */ -}} -{{- $defaultEnableCnpStatusUpdates := "true" -}} {{- $defaultBpfMapDynamicSizeRatio := 0.0 -}} {{- $defaultBpfMasquerade := "false" -}} {{- $defaultBpfClockProbe := "false" -}} @@ -13,10 +12,12 @@ {{- $fragmentTracking := "true" -}} {{- $defaultKubeProxyReplacement := "false" -}} {{- $azureUsePrimaryAddress := "true" -}} +{{- $defaultK8sClientQPS := 5 -}} +{{- $defaultK8sClientBurst := 10 -}} +{{- $defaultDNSProxyEnableTransparentMode := "false" -}} {{- /* Default values when 1.8 was initially deployed */ -}} {{- if semverCompare ">=1.8" (default "1.8" .Values.upgradeCompatibility) -}} - {{- $defaultEnableCnpStatusUpdates = "false" -}} {{- $defaultBpfMapDynamicSizeRatio = 0.0025 -}} {{- $defaultBpfMasquerade = "true" -}} {{- $defaultBpfClockProbe = "true" -}} @@ -48,6 +49,7 @@ {{- $azureUsePrimaryAddress = "false" -}} {{- end }} {{- $defaultKubeProxyReplacement = "disabled" -}} + {{- $defaultDNSProxyEnableTransparentMode = "true" -}} {{- end -}} {{- /* Default values when 1.14 was initially deployed */ -}} @@ -76,6 +78,11 @@ {{- else if (not (kindIs "invalid" .Values.cni.chainingTarget)) -}} {{- $cniChainingMode = "generic-veth" -}} {{- end -}} + +{{- if semverCompare ">=1.27-0" .Capabilities.KubeVersion.Version -}} + {{- $defaultK8sClientQPS = 10 -}} + {{- $defaultK8sClientBurst = 20 -}} +{{- end -}} --- apiVersion: v1 kind: ConfigMap @@ -359,6 +366,11 @@ data: enable-host-legacy-routing: "true" {{- end }} +{{- if .Values.bpf.nodeMapMax }} + # node-map-max specifies the maximum number of entries for the node map. + bpf-node-map-max: {{ .Values.bpf.nodeMapMax | quote }} +{{- end }} + {{- if .Values.bpf.authMapMax }} # bpf-auth-map-max specifies the maximum number of entries in the auth map bpf-auth-map-max: {{ .Values.bpf.authMapMax | quote }} @@ -443,27 +455,23 @@ data: # - vxlan (default) # - geneve {{- if .Values.gke.enabled }} + {{- if ne (.Values.routingMode | default "native") "native" }} + {{- fail (printf "RoutingMode must be set to native when gke.enabled=true" )}} + {{- end }} routing-mode: "native" enable-endpoint-routes: "true" {{- else if .Values.aksbyocni.enabled }} + {{- if ne (.Values.routingMode | default "tunnel") "tunnel" }} + {{- fail (printf "RoutingMode must be set to tunnel when aksbyocni.enabled=true" )}} + {{- end }} routing-mode: "tunnel" tunnel-protocol: "vxlan" {{- else if .Values.routingMode }} routing-mode: {{ .Values.routingMode | quote }} {{- else }} - {{- if eq .Values.tunnel "disabled" }} - routing-mode: "native" - {{- else if eq .Values.tunnel "vxlan" }} - routing-mode: "tunnel" - tunnel-protocol: "vxlan" - {{- else if eq .Values.tunnel "geneve" }} - routing-mode: "tunnel" - tunnel-protocol: "geneve" - {{- else }} # Default case routing-mode: "tunnel" tunnel-protocol: "vxlan" - {{- end }} {{- end }} {{- if .Values.tunnelProtocol }} @@ -474,6 +482,10 @@ data: tunnel-port: {{ .Values.tunnelPort | quote }} {{- end }} +{{- if .Values.serviceNoBackendResponse }} + service-no-backend-response: "{{ .Values.serviceNoBackendResponse }}" +{{- end}} + {{- if .Values.MTU }} mtu: {{ .Values.MTU | quote }} {{- end }} @@ -819,6 +831,9 @@ data: {{- if (not (kindIs "invalid" .Values.cni.chainingTarget)) }} cni-chaining-target: {{ .Values.cni.chainingTarget | quote }} {{- end}} +{{- if (not (kindIs "invalid" .Values.cni.externalRouting)) }} + cni-external-routing: {{ .Values.cni.externalRouting | quote }} +{{- end}} {{- if .Values.kubeConfigPath }} k8s-kubeconfig-path: {{ .Values.kubeConfigPath | quote }} {{- end }} @@ -883,6 +898,8 @@ data: {{- if .Values.hubble.redact.http }} # Enables redaction of the http URL query part in flows hubble-redact-http-urlquery: {{ .Values.hubble.redact.http.urlQuery | quote }} + # Enables redaction of the http user info in flows + hubble-redact-http-userinfo: {{ .Values.hubble.redact.http.userInfo | quote }} {{- if .Values.hubble.redact.http.headers }} {{- if .Values.hubble.redact.http.headers.allow }} # Redact all http headers that do not match this list @@ -904,6 +921,19 @@ data: {{- end }} {{- end }} {{- end }} +{{- if .Values.hubble.export }} + hubble-export-file-max-size-mb: {{ .Values.hubble.export.fileMaxSizeMb | quote }} + hubble-export-file-max-backups: {{ .Values.hubble.export.fileMaxBackups | quote }} +{{- if .Values.hubble.export.static.enabled }} + hubble-export-file-path: {{ .Values.hubble.export.static.filePath | quote }} + hubble-export-fieldmask: {{ .Values.hubble.export.static.fieldMask | join " " | quote }} + hubble-export-allowlist: {{ .Values.hubble.export.static.allowList | join "," | quote }} + hubble-export-denylist: {{ .Values.hubble.export.static.denyList | join "," | quote }} +{{- end }} +{{- if .Values.hubble.export.dynamic.enabled }} + hubble-flowlogs-config-path: /flowlog-config/flowlogs.yaml +{{- end }} +{{- end }} {{- if hasKey .Values.hubble "listenAddress" }} # An additional address for Hubble server to listen to (e.g. ":4244"). hubble-listen-address: {{ .Values.hubble.listenAddress | quote }} @@ -983,13 +1013,6 @@ data: api-rate-limit: {{ .Values.apiRateLimit | quote }} {{- end }} -{{- if .Values.enableCnpStatusUpdates }} - disable-cnp-status-updates: "false" -{{- else if (eq $defaultEnableCnpStatusUpdates "false") }} - disable-cnp-status-updates: "true" - cnp-node-status-gc-interval: "0s" -{{- end }} - {{- if .Values.egressGateway.enabled }} enable-ipv4-egress-gateway: "true" {{- end }} @@ -1019,10 +1042,6 @@ data: {{- end }} {{- end }} -{{- if .Values.enableK8sEventHandover }} - enable-k8s-event-handover: "true" -{{- end }} - {{- if .Values.crdWaitTimeout }} crd-wait-timeout: {{ include "validateDuration" .Values.crdWaitTimeout | quote }} {{- end }} @@ -1121,10 +1140,8 @@ data: annotate-k8s-node: "true" {{- end }} -{{- if hasKey .Values "k8sClientRateLimit" }} - k8s-client-qps: {{ .Values.k8sClientRateLimit.qps | quote }} - k8s-client-burst: {{ .Values.k8sClientRateLimit.burst | quote }} -{{- end }} + k8s-client-qps: {{ .Values.k8sClientRateLimit.qps | default $defaultK8sClientQPS | quote}} + k8s-client-burst: {{ .Values.k8sClientRateLimit.burst | default $defaultK8sClientBurst | quote }} {{- if and .Values.operator.setNodeTaints (not .Values.operator.removeNodeTaints) -}} {{ fail "Cannot have operator.setNodeTaintsMaxNodes and not operator.removeNodeTaints = false" }} @@ -1149,6 +1166,16 @@ data: {{- end }} {{- if .Values.dnsProxy }} + {{- if hasKey .Values.dnsProxy "enableTransparentMode" }} + # explicit setting gets precedence + dnsproxy-enable-transparent-mode: {{ .Values.dnsProxy.enableTransparentMode | quote }} + {{- else if eq $cniChainingMode "none" }} + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: {{ $defaultDNSProxyEnableTransparentMode | quote }} + {{- end }} + {{- if (not (kindIs "invalid" .Values.dnsProxy.socketLingerTimeout)) }} + dnsproxy-socket-linger-timeout: {{ .Values.dnsProxy.socketLingerTimeout | quote }} + {{- end }} {{- if .Values.dnsProxy.dnsRejectResponseCode }} tofqdns-dns-reject-response-code: {{ .Values.dnsProxy.dnsRejectResponseCode | quote }} {{- end }} @@ -1202,9 +1229,12 @@ data: mesh-auth-spiffe-trust-domain: {{ .Values.authentication.mutual.spire.trustDomain | quote }} {{- end }} + proxy-xff-num-trusted-hops-ingress: {{ .Values.envoy.xffNumTrustedHopsL7PolicyIngress | quote }} + proxy-xff-num-trusted-hops-egress: {{ .Values.envoy.xffNumTrustedHopsL7PolicyEgress | quote }} proxy-connect-timeout: {{ .Values.envoy.connectTimeoutSeconds | quote }} proxy-max-requests-per-connection: {{ .Values.envoy.maxRequestsPerConnection | quote }} proxy-max-connection-duration-seconds: {{ .Values.envoy.maxConnectionDurationSeconds | quote }} + proxy-idle-timeout-seconds: {{ .Values.envoy.idleTimeoutDurationSeconds | quote }} external-envoy-proxy: {{ .Values.envoy.enabled | quote }} @@ -1212,6 +1242,10 @@ data: envoy-log: {{ .Values.envoy.log.path | quote }} {{- end }} +{{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + max-connected-clusters: {{ .Values.clustermesh.maxConnectedClusters | quote }} +{{- end }} + # Extra config allows adding arbitrary properties to the cilium config. # By putting it at the end of the ConfigMap, it's also possible to override existing properties. {{- if .Values.extraConfig }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/configmap.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/configmap.yaml index 990cf951a..4b6b9218f 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/configmap.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/configmap.yaml @@ -11,6 +11,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: -{{- (tpl (.Files.Glob "files/cilium-envoy/configmap/bootstrap-config.json").AsConfig .) | nindent 2 }} - + # Keep the key name as bootstrap-config.json to avoid breaking changes + bootstrap-config.json: | + {{- (tpl (.Files.Get "files/cilium-envoy/configmap/bootstrap-config.yaml") .) | fromYaml | toJson | nindent 4 }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/daemonset.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/daemonset.yaml index 1e099aab9..d20e383f5 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/daemonset.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/daemonset.yaml @@ -26,20 +26,18 @@ spec: template: metadata: annotations: - {{- if and .Values.proxy.prometheus.enabled .Values.envoy.prometheus.enabled (not .Values.envoy.prometheus.serviceMonitor.enabled) }} - prometheus.io/port: "{{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port }}" - prometheus.io/scrape: "true" - {{- end }} {{- if .Values.envoy.rollOutPods }} # ensure pods roll when configmap updates cilium.io/cilium-envoy-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-envoy/configmap.yaml") . | sha256sum | quote }} {{- end }} {{- if not .Values.envoy.securityContext.privileged }} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} # Set app AppArmor's profile to "unconfined". The value of this annotation # can be modified as long users know which profiles they have available # in AppArmor. container.apparmor.security.beta.kubernetes.io/cilium-envoy: "unconfined" {{- end }} + {{- end }} {{- with .Values.envoy.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -56,6 +54,11 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- /* K8s version lower than 1.30.0 don't support the "appArmorProfile" field, */}} + {{- /* thus we have to remove it. */}} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} + {{- $_ := unset .Values.envoy.podSecurityContext "appArmorProfile" }} + {{- end }} {{- with .Values.envoy.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} @@ -83,44 +86,32 @@ spec: {{- with .Values.envoy.extraArgs }} {{- toYaml . | trim | nindent 8 }} {{- end }} - {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} startupProbe: httpGet: - host: "localhost" + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} path: /healthz port: {{ .Values.envoy.healthPort }} scheme: HTTP failureThreshold: {{ .Values.envoy.startupProbe.failureThreshold }} periodSeconds: {{ .Values.envoy.startupProbe.periodSeconds }} successThreshold: 1 - {{- end }} + initialDelaySeconds: 5 livenessProbe: httpGet: - host: "localhost" + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} path: /healthz port: {{ .Values.envoy.healthPort }} scheme: HTTP - {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} - # The initial delay for the liveness probe is intentionally large to - # avoid an endless kill & restart cycle if in the event that the initial - # bootstrapping takes longer than expected. - # Starting from Kubernetes 1.20, we are using startupProbe instead - # of this field. - initialDelaySeconds: 120 - {{- end }} periodSeconds: {{ .Values.envoy.livenessProbe.periodSeconds }} successThreshold: 1 failureThreshold: {{ .Values.envoy.livenessProbe.failureThreshold }} timeoutSeconds: 5 readinessProbe: httpGet: - host: "localhost" + host: {{ .Values.ipv4.enabled | ternary "127.0.0.1" "::1" | quote }} path: /healthz port: {{ .Values.envoy.healthPort }} scheme: HTTP - {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} - initialDelaySeconds: 5 - {{- end }} periodSeconds: {{ .Values.envoy.readinessProbe.periodSeconds }} successThreshold: 1 failureThreshold: {{ .Values.envoy.readinessProbe.failureThreshold }} @@ -206,7 +197,6 @@ spec: {{- end }} restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.envoy.priorityClassName "system-node-critical") }} - serviceAccount: {{ .Values.serviceAccounts.envoy.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.envoy.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.envoy.automount }} terminationGracePeriodSeconds: {{ .Values.envoy.terminationGracePeriodSeconds }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/service.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/service.yaml index 4628259db..d238c62e6 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/service.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/service.yaml @@ -6,13 +6,13 @@ metadata: namespace: {{ .Release.Namespace }} {{- if or (not .Values.envoy.prometheus.serviceMonitor.enabled) .Values.envoy.annotations }} annotations: - {{- if not .Values.envoy.prometheus.serviceMonitor.enabled }} - prometheus.io/scrape: "true" - prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} - {{- end }} - {{- with .Values.envoy.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} + {{- if not .Values.envoy.prometheus.serviceMonitor.enabled }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} + {{- end }} + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} labels: k8s-app: cilium-envoy diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/servicemonitor.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/servicemonitor.yaml index 3d6b745e3..10f84d82b 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-envoy/servicemonitor.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-envoy/servicemonitor.yaml @@ -7,15 +7,16 @@ metadata: namespace: {{ .Values.envoy.prometheus.serviceMonitor.namespace | default .Release.Namespace }} labels: app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy {{- with .Values.envoy.prometheus.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} - {{- if or .Values.envoy.prometheus.serviceMonitor .Values.envoy.annotations }} + {{- if or .Values.envoy.prometheus.serviceMonitor.annotations .Values.envoy.annotations }} annotations: {{- with .Values.envoy.annotations }} {{- toYaml . | nindent 4 }} {{- end }} - {{- with .Values.envoy.prometheus.serviceMonitor }} + {{- with .Values.envoy.prometheus.serviceMonitor.annotations }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-flowlog-configmap.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-flowlog-configmap.yaml new file mode 100644 index 000000000..8a1341e4a --- /dev/null +++ b/internal/constellation/helm/charts/cilium/templates/cilium-flowlog-configmap.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.hubble.export.dynamic.enabled .Values.hubble.export.dynamic.config.createConfigMap }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.hubble.export.dynamic.config.configMapName }} + namespace: {{ .Release.Namespace }} +data: + flowlogs.yaml: | + flowLogs: +{{ .Values.hubble.export.dynamic.config.content | toYaml | indent 4 }} +{{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-ingress-service.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-ingress-service.yaml index ff6269d22..0e489bdac 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-ingress-service.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-ingress-service.yaml @@ -24,14 +24,12 @@ spec: protocol: TCP nodePort: {{ .Values.ingressController.service.secureNodePort }} type: {{ .Values.ingressController.service.type }} - {{- if semverCompare ">=1.24-0" .Capabilities.KubeVersion.Version -}} {{- if .Values.ingressController.service.loadBalancerClass }} loadBalancerClass: {{ .Values.ingressController.service.loadBalancerClass }} {{- end }} {{- if (not (kindIs "invalid" .Values.ingressController.service.allocateLoadBalancerNodePorts)) }} allocateLoadBalancerNodePorts: {{ .Values.ingressController.service.allocateLoadBalancerNodePorts }} {{- end }} - {{- end -}} {{- if .Values.ingressController.service.loadBalancerIP }} loadBalancerIP: {{ .Values.ingressController.service.loadBalancerIP }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-nodeinit/daemonset.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-nodeinit/daemonset.yaml index 76f1a20d2..c92eabfa6 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-nodeinit/daemonset.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-nodeinit/daemonset.yaml @@ -28,11 +28,13 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} {{- if not .Values.securityContext.privileged }} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} # Set app AppArmor's profile to "unconfined". The value of this annotation # can be modified as long users know which profiles they have available # in AppArmor. container.apparmor.security.beta.kubernetes.io/node-init: "unconfined" {{- end }} + {{- end }} labels: app: cilium-node-init app.kubernetes.io/part-of: cilium @@ -45,6 +47,15 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- /* K8s version lower than 1.30.0 don't support the "appArmorProfile" field, */}} + {{- /* thus we have to remove it. */}} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} + {{- $_ := unset .Values.nodeinit.podSecurityContext "appArmorProfile" }} + {{- end }} + {{- with .Values.nodeinit.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: node-init image: {{ include "cilium.image" .Values.nodeinit.image | quote }} @@ -103,7 +114,6 @@ spec: hostNetwork: true priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.nodeinit.priorityClassName "system-node-critical") }} {{- if .Values.serviceAccounts.nodeinit.enabled }} - serviceAccount: {{ .Values.serviceAccounts.nodeinit.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.nodeinit.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.nodeinit.automount }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml index 66a0b888a..c4b90a273 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-operator/dashboards-configmap.yaml @@ -20,7 +20,7 @@ metadata: {{- with $.Values.operator.dashboards.annotations }} {{- toYaml . | nindent 4 }} {{- end }} - {{- with .Values.operator.annotations }} + {{- with $.Values.operator.annotations }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-operator/deployment.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-operator/deployment.yaml index 4f4450e51..5c6c467cf 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-operator/deployment.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-operator/deployment.yaml @@ -252,7 +252,6 @@ spec: {{- end }} restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.operator.priorityClassName "system-cluster-critical") }} - serviceAccount: {{ .Values.serviceAccounts.operator.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.operator.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.operator.automount }} {{- with .Values.operator.affinity }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml index a224b9e6c..05b251046 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-operator/poddisruptionbudget.yaml @@ -1,6 +1,6 @@ {{- if and .Values.operator.enabled .Values.operator.podDisruptionBudget.enabled }} {{- $component := .Values.operator.podDisruptionBudget }} -apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: cilium-operator diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-preflight/daemonset.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-preflight/daemonset.yaml index bc13be432..b5228616b 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-preflight/daemonset.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-preflight/daemonset.yaml @@ -70,8 +70,13 @@ spec: - /tmp/ready initialDelaySeconds: 5 periodSeconds: 5 - {{- with .Values.preflight.extraEnv }} env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + {{- with .Values.preflight.extraEnv }} {{- toYaml . | trim | nindent 12 }} {{- end }} volumeMounts: @@ -171,10 +176,13 @@ spec: dnsPolicy: ClusterFirstWithHostNet restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.preflight.priorityClassName "system-node-critical") }} - serviceAccount: {{ .Values.serviceAccounts.preflight.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.preflight.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.preflight.automount }} terminationGracePeriodSeconds: {{ .Values.preflight.terminationGracePeriodSeconds }} + {{- with .Values.preflight.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.preflight.tolerations }} tolerations: {{- toYaml . | trim | nindent 8 }} diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-preflight/deployment.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-preflight/deployment.yaml index efd923b2d..1f87d2076 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-preflight/deployment.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-preflight/deployment.yaml @@ -60,6 +60,10 @@ spec: - /tmp/ready-validate-cnp initialDelaySeconds: 5 periodSeconds: 5 + {{- with .Values.preflight.extraVolumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} env: {{- if .Values.k8sServiceHost }} - name: KUBERNETES_SERVICE_HOST @@ -77,11 +81,15 @@ spec: {{- toYaml . | trim | nindent 12 }} {{- end }} terminationMessagePolicy: FallbackToLogsOnError + {{- with .Values.preflight.extraVolumes }} + volumes: + {{- toYaml . | trim | nindent 6 }} + {{- end }} hostNetwork: true restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.preflight.priorityClassName "system-cluster-critical") }} - serviceAccount: {{ .Values.serviceAccounts.preflight.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.preflight.name | quote }} + automountServiceAccountToken: {{ .Values.serviceAccounts.preflight.automount }} terminationGracePeriodSeconds: {{ .Values.preflight.terminationGracePeriodSeconds }} {{- with .Values.preflight.affinity }} affinity: diff --git a/internal/constellation/helm/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml index 4b3c7cb0d..c00d9b896 100644 --- a/internal/constellation/helm/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cilium/templates/cilium-preflight/poddisruptionbudget.yaml @@ -1,6 +1,6 @@ {{- if and .Values.preflight.enabled .Values.preflight.validateCNPs .Values.preflight.podDisruptionBudget.enabled }} {{- $component := .Values.preflight.podDisruptionBudget }} -apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: cilium-pre-flight-check diff --git a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/deployment.yaml b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/deployment.yaml index b871ae796..f0d551bb6 100644 --- a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/deployment.yaml +++ b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/deployment.yaml @@ -48,44 +48,51 @@ spec: {{- end }} initContainers: - name: etcd-init - image: {{ include "cilium.image" .Values.clustermesh.apiserver.etcd.image | quote }} - imagePullPolicy: {{ .Values.clustermesh.apiserver.etcd.image.pullPolicy }} - command: ["/bin/sh", "-c"] + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} + command: + - /usr/bin/clustermesh-apiserver args: - - | - rm -rf /var/run/etcd/*; - /usr/local/bin/etcd --data-dir=/var/run/etcd --name=clustermesh-apiserver --listen-client-urls=http://127.0.0.1:2379 --advertise-client-urls=http://127.0.0.1:2379 --initial-cluster-token=clustermesh-apiserver --initial-cluster-state=new --auto-compaction-retention=1 & - - # The following key needs to be created before that the cilium agents - # have the possibility of connecting to etcd. - etcdctl put cilium/.has-cluster-config true - - etcdctl user add root --no-password; - etcdctl user grant-role root root; - etcdctl user add admin-{{ .Values.cluster.name }} --no-password; - etcdctl user grant-role admin-{{ .Values.cluster.name }} root; - etcdctl user add externalworkload --no-password; - etcdctl role add externalworkload; - etcdctl role grant-permission externalworkload --from-key read ''; - etcdctl role grant-permission externalworkload readwrite --prefix cilium/state/noderegister/v1/; - etcdctl role grant-permission externalworkload readwrite --prefix cilium/.initlock/; - etcdctl user grant-role externalworkload externalworkload; - etcdctl user add remote --no-password; - etcdctl role add remote; - etcdctl role grant-permission remote --from-key read ''; - etcdctl user grant-role remote remote; - etcdctl auth enable; - exit + - etcdinit + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + # These need to match the equivalent arguments to etcd in the main container. + - --etcd-cluster-name=clustermesh-apiserver + - --etcd-initial-cluster-token=$(INITIAL_CLUSTER_TOKEN) + - --etcd-data-dir=/var/run/etcd + {{- with .Values.clustermesh.apiserver.etcd.init.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} env: - - name: ETCDCTL_API - value: "3" - - name: HOSTNAME_IP + # The Cilium cluster name (specified via the `CILIUM_CLUSTER_NAME` environment variable) and the etcd cluster + # name (specified via the `--etcd-cluster-name` argument) are very different concepts. The Cilium cluster name + # is the name of the overall Cilium cluster, and is used to set the admin account username. The etcd cluster + # name is a concept that's only relevant for etcd itself. The etcd cluster name must be the same for both this + # command and the actual invocation of etcd in the main containers of this Pod, but it's otherwise not + # relevant to Cilium. + - name: CILIUM_CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cilium-config + key: cluster-name + - name: INITIAL_CLUSTER_TOKEN valueFrom: fieldRef: - fieldPath: status.podIP + fieldPath: metadata.uid + {{- with .Values.clustermesh.apiserver.etcd.init.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.etcd.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} volumeMounts: - name: etcd-data-dir mountPath: /var/run/etcd + {{- with .Values.clustermesh.apiserver.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} terminationMessagePolicy: FallbackToLogsOnError {{- with .Values.clustermesh.apiserver.etcd.init.resources }} resources: @@ -93,10 +100,11 @@ spec: {{- end }} containers: - name: etcd - image: {{ include "cilium.image" .Values.clustermesh.apiserver.etcd.image | quote }} - imagePullPolicy: {{ .Values.clustermesh.apiserver.etcd.image.pullPolicy }} + # The clustermesh-apiserver container image includes an etcd binary. + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} command: - - /usr/local/bin/etcd + - /usr/bin/etcd args: - --data-dir=/var/run/etcd - --name=clustermesh-apiserver @@ -108,7 +116,7 @@ spec: # uses net.SplitHostPort() internally and it accepts the that format. - --listen-client-urls=https://127.0.0.1:2379,https://[$(HOSTNAME_IP)]:2379 - --advertise-client-urls=https://[$(HOSTNAME_IP)]:2379 - - --initial-cluster-token=clustermesh-apiserver + - --initial-cluster-token=$(INITIAL_CLUSTER_TOKEN) - --auto-compaction-retention=1 {{- if .Values.clustermesh.apiserver.metrics.etcd.enabled }} - --listen-metrics-urls=http://[$(HOSTNAME_IP)]:{{ .Values.clustermesh.apiserver.metrics.etcd.port }} @@ -121,6 +129,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: INITIAL_CLUSTER_TOKEN + valueFrom: + fieldRef: + fieldPath: metadata.uid ports: - name: etcd containerPort: 2379 @@ -136,6 +148,9 @@ spec: readOnly: true - name: etcd-data-dir mountPath: /var/run/etcd + {{- with .Values.clustermesh.apiserver.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} terminationMessagePolicy: FallbackToLogsOnError {{- with .Values.clustermesh.apiserver.etcd.resources }} resources: @@ -155,6 +170,7 @@ spec: command: - /usr/bin/clustermesh-apiserver args: + - clustermesh {{- if .Values.debug.enabled }} - --debug {{- end }} @@ -162,6 +178,9 @@ spec: - --cluster-id=$(CLUSTER_ID) - --kvstore-opt - etcd.config=/var/lib/cilium/etcd-config.yaml + {{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + - --max-connected-clusters={{ .Values.clustermesh.maxConnectedClusters }} + {{- end }} {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} - --cluster-users-enabled - --cluster-users-config-path=/var/lib/cilium/etcd-config/users.yaml @@ -233,11 +252,12 @@ spec: {{- end }} {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} - name: kvstoremesh - image: {{ include "cilium.image" .Values.clustermesh.apiserver.kvstoremesh.image | quote }} - imagePullPolicy: {{ .Values.clustermesh.apiserver.kvstoremesh.image.pullPolicy }} + image: {{ include "cilium.image" .Values.clustermesh.apiserver.image | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} command: - - /usr/bin/kvstoremesh + - /usr/bin/clustermesh-apiserver args: + - kvstoremesh {{- if .Values.debug.enabled }} - --debug {{- end }} @@ -247,6 +267,9 @@ spec: - --kvstore-opt=etcd.qps=100 - --kvstore-opt=etcd.maxInflight=10 - --clustermesh-config=/var/lib/cilium/clustermesh + {{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + - --max-connected-clusters={{ .Values.clustermesh.maxConnectedClusters }} + {{- end }} {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} - --controller-group-metrics=all @@ -381,7 +404,6 @@ spec: {{- end }} restartPolicy: Always priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.clustermesh.apiserver.priorityClassName "system-cluster-critical") }} - serviceAccount: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} terminationGracePeriodSeconds: {{ .Values.clustermesh.apiserver.terminationGracePeriodSeconds }} automountServiceAccountToken: {{ .Values.serviceAccounts.clustermeshApiserver.automount }} diff --git a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml index 4a1bbf7e0..a5d30b7b1 100644 --- a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml @@ -1,6 +1,6 @@ {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.podDisruptionBudget.enabled }} {{- $component := .Values.clustermesh.apiserver.podDisruptionBudget }} -apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: clustermesh-apiserver diff --git a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/service.yaml b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/service.yaml index 0a7028c54..14daaeb59 100644 --- a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/service.yaml +++ b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/service.yaml @@ -26,6 +26,9 @@ spec: {{- if and (eq "NodePort" .Values.clustermesh.apiserver.service.type) .Values.clustermesh.apiserver.service.nodePort }} nodePort: {{ .Values.clustermesh.apiserver.service.nodePort }} {{- end }} + {{- if and (eq "LoadBalancer" .Values.clustermesh.apiserver.service.type) .Values.clustermesh.apiserver.service.loadBalancerClass }} + loadBalancerClass: {{ .Values.clustermesh.apiserver.service.loadBalancerClass }} + {{- end }} {{- if and (eq "LoadBalancer" .Values.clustermesh.apiserver.service.type) .Values.clustermesh.apiserver.service.loadBalancerIP }} loadBalancerIP: {{ .Values.clustermesh.apiserver.service.loadBalancerIP }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml index 946602b40..8c0e4cd5c 100644 --- a/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml +++ b/internal/constellation/helm/charts/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml @@ -1,5 +1,5 @@ {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") .Values.clustermesh.apiserver.tls.auto.schedule }} -apiVersion: {{ include "cronjob.apiVersion" . }} +apiVersion: batch/v1 kind: CronJob metadata: name: clustermesh-apiserver-generate-certs diff --git a/internal/constellation/helm/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml b/internal/constellation/helm/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml index 5946219f4..7aefc0d35 100644 --- a/internal/constellation/helm/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml +++ b/internal/constellation/helm/charts/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml @@ -110,7 +110,6 @@ spec: hostNetwork: true priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.clustermesh.apiserver.priorityClassName "system-cluster-critical") }} restartPolicy: Always - serviceAccount: {{ .Values.serviceAccounts.etcd.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.etcd.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.etcd.automount }} {{- with .Values.etcd.nodeSelector }} diff --git a/internal/constellation/helm/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml index 5939b4ae9..d604e5222 100644 --- a/internal/constellation/helm/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cilium/templates/etcd-operator/poddisruptionbudget.yaml @@ -1,6 +1,6 @@ {{- if and .Values.etcd.managed .Values.etcd.podDisruptionBudget.enabled }} {{- $component := .Values.etcd.podDisruptionBudget }} -apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: cilium-etcd-operator diff --git a/internal/constellation/helm/charts/cilium/templates/hubble-relay/deployment.yaml b/internal/constellation/helm/charts/cilium/templates/hubble-relay/deployment.yaml index 6f1ec8d0f..5a5fb35a8 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble-relay/deployment.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble-relay/deployment.yaml @@ -71,11 +71,37 @@ spec: protocol: TCP {{- end }} readinessProbe: - tcpSocket: - port: grpc + grpc: + port: 4222 + timeoutSeconds: 3 + # livenessProbe will kill the pod, we should be very conservative + # here on failures since killing the pod should be a last resort, and + # we should provide enough time for relay to retry before killing it. livenessProbe: - tcpSocket: - port: grpc + grpc: + port: 4222 + timeoutSeconds: 10 + # Give relay time to establish connections and make a few retries + # before starting livenessProbes. + initialDelaySeconds: 10 + # 10 second * 12 failures = 2 minutes of failure. + # If relay cannot become healthy after 2 minutes, then killing it + # might resolve whatever issue is occurring. + # + # 10 seconds is a reasonable retry period so we can see if it's + # failing regularly or only sporadically. + periodSeconds: 10 + failureThreshold: 12 + startupProbe: + grpc: + port: 4222 + # Give relay time to get it's certs and establish connections and + # make a few retries before starting startupProbes. + initialDelaySeconds: 10 + # 20 * 3 seconds = 1 minute of failure before we consider startup as failed. + failureThreshold: 20 + # Retry more frequently at startup so that it can be considered started more quickly. + periodSeconds: 3 {{- with .Values.hubble.relay.extraEnv }} env: {{- toYaml . | trim | nindent 12 }} @@ -93,10 +119,12 @@ spec: mountPath: /var/lib/hubble-relay/tls readOnly: true {{- end }} + {{- with .Values.hubble.relay.extraVolumeMounts }} + {{- toYaml . | nindent 10 }} + {{- end }} terminationMessagePolicy: FallbackToLogsOnError restartPolicy: Always priorityClassName: {{ .Values.hubble.relay.priorityClassName }} - serviceAccount: {{ .Values.serviceAccounts.relay.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.relay.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.relay.automount }} terminationGracePeriodSeconds: {{ .Values.hubble.relay.terminationGracePeriodSeconds }} @@ -163,4 +191,7 @@ spec: path: server.key {{- end }} {{- end }} + {{- with .Values.hubble.relay.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml index 4fd6da9ba..6162cb81d 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble-relay/poddisruptionbudget.yaml @@ -1,6 +1,6 @@ {{- if and .Values.hubble.enabled .Values.hubble.relay.enabled .Values.hubble.relay.podDisruptionBudget.enabled }} {{- $component := .Values.hubble.relay.podDisruptionBudget }} -apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: hubble-relay diff --git a/internal/constellation/helm/charts/cilium/templates/hubble-ui/_nginx.tpl b/internal/constellation/helm/charts/cilium/templates/hubble-ui/_nginx.tpl index e787b5aad..5d3d0a80e 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble-ui/_nginx.tpl +++ b/internal/constellation/helm/charts/cilium/templates/hubble-ui/_nginx.tpl @@ -13,24 +13,12 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - # CORS - add_header Access-Control-Allow-Methods "GET, POST, PUT, HEAD, DELETE, OPTIONS"; - add_header Access-Control-Allow-Origin *; - add_header Access-Control-Max-Age 1728000; - add_header Access-Control-Expose-Headers content-length,grpc-status,grpc-message; - add_header Access-Control-Allow-Headers range,keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout; - if ($request_method = OPTIONS) { - return 204; - } - # /CORS - location {{ .Values.hubble.ui.baseUrl }}api { {{- if not (eq .Values.hubble.ui.baseUrl "/") }} rewrite ^{{ (trimSuffix "/" .Values.hubble.ui.baseUrl) }}(/.*)$ $1 break; {{- end }} proxy_http_version 1.1; proxy_pass_request_headers on; - proxy_hide_header Access-Control-Allow-Origin; {{- if eq .Values.hubble.ui.baseUrl "/" }} proxy_pass http://127.0.0.1:8090; {{- else }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble-ui/deployment.yaml b/internal/constellation/helm/charts/cilium/templates/hubble-ui/deployment.yaml index a7dd5cb8f..105907a5f 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble-ui/deployment.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble-ui/deployment.yaml @@ -40,13 +40,10 @@ spec: {{- end }} spec: {{- with .Values.hubble.ui.securityContext }} - {{- if .enabled }} securityContext: {{- omit . "enabled" | toYaml | nindent 8 }} - {{- end}} {{- end }} priorityClassName: {{ .Values.hubble.ui.priorityClassName }} - serviceAccount: {{ .Values.serviceAccounts.ui.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.ui.name | quote }} automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }} {{- with .Values.imagePullSecrets }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble-ui/ingress.yaml b/internal/constellation/helm/charts/cilium/templates/hubble-ui/ingress.yaml index 2c0ff7d3e..348e281d7 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble-ui/ingress.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble-ui/ingress.yaml @@ -1,6 +1,6 @@ {{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.hubble.ui.ingress.enabled }} {{- $baseUrl := .Values.hubble.ui.baseUrl -}} -apiVersion: {{ template "ingress.apiVersion" . }} +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: hubble-ui @@ -35,6 +35,11 @@ spec: http: paths: - path: {{ $baseUrl | quote }} - {{- include "ingress.paths" $ | nindent 12 }} + pathType: Prefix + backend: + service: + name: hubble-ui + port: + name: http {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml b/internal/constellation/helm/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml index af3b6705d..c23e3ad04 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble-ui/poddisruptionbudget.yaml @@ -1,6 +1,6 @@ {{- if and (or .Values.hubble.enabled .Values.hubble.ui.standalone.enabled) .Values.hubble.ui.enabled .Values.hubble.ui.podDisruptionBudget.enabled }} {{- $component := .Values.hubble.ui.podDisruptionBudget }} -apiVersion: {{ include "podDisruptionBudget.apiVersion" . }} +apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: hubble-ui diff --git a/internal/constellation/helm/charts/cilium/templates/hubble/peer-service.yaml b/internal/constellation/helm/charts/cilium/templates/hubble/peer-service.yaml index 7ba56456b..aec3f889a 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble/peer-service.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble/peer-service.yaml @@ -24,7 +24,5 @@ spec: {{- end }} protocol: TCP targetPort: {{ .Values.hubble.peerService.targetPort }} -{{- if semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion }} internalTrafficPolicy: Local {{- end }} -{{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml index 1dd96b18c..373d6c541 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml @@ -19,4 +19,9 @@ spec: duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} privateKey: rotationPolicy: Always + isCA: false + usages: + - signing + - key encipherment + - client auth {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml index 845b4fb8e..c33b912b1 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml @@ -28,4 +28,9 @@ spec: duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} privateKey: rotationPolicy: Always + isCA: false + usages: + - signing + - key encipherment + - server auth {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml index 5f202e10b..b34f27c52 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/server-secret.yaml @@ -29,4 +29,10 @@ spec: duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} privateKey: rotationPolicy: Always + isCA: false + usages: + - signing + - key encipherment + - server auth + - client auth {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml index 5006666ec..64ace1872 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml @@ -19,4 +19,9 @@ spec: duration: {{ printf "%dh0m0s" (mul .Values.hubble.tls.auto.certValidityDuration 24) }} privateKey: rotationPolicy: Always + isCA: false + usages: + - signing + - key encipherment + - client auth {{- end }} diff --git a/internal/constellation/helm/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml b/internal/constellation/helm/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml index fa9966080..7d9f7174c 100644 --- a/internal/constellation/helm/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml +++ b/internal/constellation/helm/charts/cilium/templates/hubble/tls-cronjob/cronjob.yaml @@ -1,5 +1,5 @@ {{- if and .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "cronJob") .Values.hubble.tls.auto.schedule }} -apiVersion: {{ include "cronjob.apiVersion" . }} +apiVersion: batch/v1 kind: CronJob metadata: name: hubble-generate-certs diff --git a/internal/constellation/helm/charts/cilium/templates/spire/agent/daemonset.yaml b/internal/constellation/helm/charts/cilium/templates/spire/agent/daemonset.yaml index 933492877..6c0bffe78 100644 --- a/internal/constellation/helm/charts/cilium/templates/spire/agent/daemonset.yaml +++ b/internal/constellation/helm/charts/cilium/templates/spire/agent/daemonset.yaml @@ -35,6 +35,10 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet serviceAccountName: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.install.agent.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} initContainers: - name: init image: {{ include "cilium.image" .Values.authentication.mutual.spire.install.initImage | quote }} @@ -53,6 +57,10 @@ spec: imagePullPolicy: {{ .Values.authentication.mutual.spire.install.agent.image.pullPolicy }} {{- end }} args: ["-config", "/run/spire/config/agent.conf"] + {{- with .Values.authentication.mutual.spire.install.agent.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: spire-config mountPath: /run/spire/config @@ -83,10 +91,20 @@ spec: port: 4251 initialDelaySeconds: 5 periodSeconds: 5 - {{- with .Values.authentication.mutual.spire.install.agent.tolerations }} - tolerations: - {{- toYaml . | trim | nindent 8 }} + {{- with .Values.authentication.mutual.spire.install.agent.affinity }} + affinity: + {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.authentication.mutual.spire.install.agent.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- with .Values.authentication.mutual.spire.install.agent.tolerations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + - key: {{ .Values.agentNotReadyTaintKey | default "node.cilium.io/agent-not-ready" }} + effect: NoSchedule volumes: - name: spire-config configMap: diff --git a/internal/constellation/helm/charts/cilium/templates/spire/namespace.yaml b/internal/constellation/helm/charts/cilium/templates/spire/namespace.yaml index 1c281f4f7..ccd386808 100644 --- a/internal/constellation/helm/charts/cilium/templates/spire/namespace.yaml +++ b/internal/constellation/helm/charts/cilium/templates/spire/namespace.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled (not .Values.authentication.mutual.spire.install.existingNamespace) -}} apiVersion: v1 kind: Namespace metadata: diff --git a/internal/constellation/helm/charts/cilium/templates/validate.yaml b/internal/constellation/helm/charts/cilium/templates/validate.yaml index 070dfeff4..fabd69fe9 100644 --- a/internal/constellation/helm/charts/cilium/templates/validate.yaml +++ b/internal/constellation/helm/charts/cilium/templates/validate.yaml @@ -1,3 +1,17 @@ +{{/* validate deprecated options are not being used */}} +{{- if .Values.tunnel }} + {{ fail "tunnel was deprecated in v1.14 and has been removed in v1.15. For details please refer to https://docs.cilium.io/en/v1.15/operations/upgrade/#helm-options" }} +{{- end }} +{{- if or (dig "clustermesh" "apiserver" "tls" "ca" "cert" "" .Values.AsMap) (dig "clustermesh" "apiserver" "tls" "ca" "key" "" .Values.AsMap) }} + {{ fail "clustermesh.apiserver.tls.ca.cert and clustermesh.apiserver.tls.ca.key were deprecated in v1.14 and has been removed in v1.15. For details please refer to https://docs.cilium.io/en/v1.15/operations/upgrade/#helm-options" }} +{{- end }} +{{- if .Values.enableK8sEventHandover }} + {{ fail "enableK8sEventHandover was deprecated in v1.14 and has been removed in v1.15. For details please refer to https://docs.cilium.io/en/v1.15/operations/upgrade/#helm-options" }} +{{- end }} +{{- if .Values.enableCnpStatusUpdates }} + {{ fail "enableCnpStatusUpdates was deprecated in v1.14 and has been removed in v1.15. For details please refer to https://docs.cilium.io/en/v1.15/operations/upgrade/#helm-options" }} +{{- end }} + {{/* validate hubble config */}} {{- if and .Values.hubble.ui.enabled (not .Values.hubble.ui.standalone.enabled) }} {{- if not .Values.hubble.relay.enabled }} @@ -96,3 +110,8 @@ {{ fail "External workloads support cannot be enabled in combination with .Values.disableEndpointCRD=true" }} {{- end }} {{- end }} + +{{/*validate ClusterMesh */}} +{{- if and (ne (int .Values.clustermesh.maxConnectedClusters) 255) (ne (int .Values.clustermesh.maxConnectedClusters) 511) }} + {{- fail "max-connected-clusters must be set to 255 or 511" }} +{{- end }} diff --git a/internal/constellation/helm/charts/cilium/values.yaml b/internal/constellation/helm/charts/cilium/values.yaml index 7d432c8ba..c87bdc204 100644 --- a/internal/constellation/helm/charts/cilium/values.yaml +++ b/internal/constellation/helm/charts/cilium/values.yaml @@ -47,11 +47,13 @@ k8sServicePort: "" # rate limit, the agent and operator will start to throttle requests by delaying # them until there is budget or the request times out. k8sClientRateLimit: - # -- The sustained request rate in requests per second. - qps: 5 - # -- The burst request rate in requests per second. + # -- (int) The sustained request rate in requests per second. + # @default -- 5 for k8s up to 1.26. 10 for k8s version 1.27+ + qps: + # -- (int) The burst request rate in requests per second. # The rate limiter will allow short bursts with a higher rate. - burst: 10 + # @default -- 10 for k8s up to 1.26. 20 for k8s version 1.27+ + burst: cluster: # -- Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. @@ -144,7 +146,7 @@ rollOutCiliumPods: false image: override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.15.0-pre.2" + tag: "v1.15.19" pullPolicy: "IfNotPresent" # cilium-digest digest: "" @@ -216,8 +218,10 @@ extraConfig: {} annotations: {} # -- Security Context for cilium-agent pods. -podSecurityContext: {} - +podSecurityContext: + # -- AppArmorProfile options for the `cilium-agent` and init containers + appArmorProfile: + type: "Unconfined" # -- Annotations to be added to agent pods podAnnotations: {} @@ -234,6 +238,9 @@ resources: {} # cpu: 100m # memory: 512Mi +# -- resources & limits for the agent init containers +initResources: {} + securityContext: # -- User to run the pod with # runAsUser: 0 @@ -412,9 +419,9 @@ bgpControlPlane: # -- SecretsNamespace is the namespace which BGP support will retrieve secrets from. secretsNamespace: # -- Create secrets namespace for BGP secrets. - create: true + create: false # -- The name of the secret namespace to which Cilium agents are given read access - name: cilium-bgp-secrets + name: kube-system pmtuDiscovery: # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to @@ -463,7 +470,17 @@ bpf: # @default -- `524288` neighMax: ~ + # @schema + # type: [null, integer] + # @schema + # @default -- `16384` + # -- (int) Configures the maximum number of entries for the node table. + nodeMapMax: ~ + # -- Configure the maximum number of entries in endpoint policy map (per endpoint). + # @schema + # type: [null, integer] + # @schema policyMapMax: 16384 # -- (float64) Configure auto-sizing for all BPF maps based on available memory. @@ -596,6 +613,12 @@ cni: # inside the agent pod. hostConfDirMountPath: /host/etc/cni/net.d + # -- Specifies the resources for the cni initContainer + resources: + requests: + cpu: 100m + memory: 10Mi + # -- (string) Configure how frequently garbage collection should occur for the datapath # connection tracking table. # @default -- `"0s"` @@ -673,13 +696,6 @@ enableRuntimeDeviceDetection: false # -- Limit iptables-based egress masquerading to interface selector. # egressMasqueradeInterfaces: "" -# -- Whether to enable CNP status updates. -enableCnpStatusUpdates: false - -# -- Configures the use of the KVStore to optimize Kubernetes event handling by -# mirroring it into the KVstore for reduced overhead in large clusters. -enableK8sEventHandover: false - # -- Enable setting identity mark for local traffic. # enableIdentityMark: true @@ -724,8 +740,7 @@ ingressController: # -- Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. enableProxyProtocol: false - # -- IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate - # from Ingress to the Load Balancer service + # -- IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] # -- Default secret namespace for ingresses without .spec.tls[].secretName set. @@ -966,8 +981,8 @@ certgen: image: override: ~ repository: "quay.io/cilium/certgen" - tag: "v0.1.9" - digest: "sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f" + tag: "v0.1.19" + digest: "sha256:28511366bb5dc99b6ec424dc87399945714d57a586194658d9e2316ba3db4d04" useDigest: true pullPolicy: "IfNotPresent" # -- Seconds after which the completed job pod will be deleted @@ -1086,6 +1101,19 @@ hubble: # --set hubble.redact.enabled="true" # --set hubble.redact.http.urlQuery="true" urlQuery: false + # -- Enables redacting user info, e.g., password when basic auth is used. + # Example: + # + # redact: + # enabled: true + # http: + # userInfo: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.userInfo="true" + userInfo: true headers: # -- List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. # Example: @@ -1129,7 +1157,7 @@ hubble: # # --set hubble.redact.enabled="true" # --set hubble.redact.kafka.apiKey="true" - apiKey: false + apiKey: true # -- An additional address for Hubble to listen to. # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that @@ -1212,7 +1240,7 @@ hubble: image: override: ~ repository: "quay.io/cilium/hubble-relay" - tag: "v1.15.0-pre.2" + tag: "v1.15.19" # hubble-relay-digest digest: "" useDigest: false @@ -1283,6 +1311,12 @@ hubble: rollingUpdate: maxUnavailable: 1 + # -- Additional hubble-relay volumes. + extraVolumes: [] + + # -- Additional hubble-relay volumeMounts. + extraVolumeMounts: [] + # -- hubble-relay pod security context podSecurityContext: fsGroup: 65532 @@ -1443,8 +1477,8 @@ hubble: image: override: ~ repository: "quay.io/cilium/hubble-ui-backend" - tag: "v0.12.1" - digest: "sha256:1f86f3400827a0451e6332262467f894eeb7caf0eb8779bd951e2caa9d027cbe" + tag: "v0.13.2" + digest: "sha256:a034b7e98e6ea796ed26df8f4e71f83fc16465a19d166eff67a03b822c0bfa15" useDigest: true pullPolicy: "IfNotPresent" @@ -1482,8 +1516,8 @@ hubble: image: override: ~ repository: "quay.io/cilium/hubble-ui" - tag: "v0.12.1" - digest: "sha256:9e5f81ee747866480ea1ac4630eb6975ff9227f9782b7c93919c081c33f38267" + tag: "v0.13.2" + digest: "sha256:9e37c1296b802830834cc87342a9182ccbb71ffebb711971e849221bd9d59392" useDigest: true pullPolicy: "IfNotPresent" @@ -1597,6 +1631,55 @@ hubble: # hosts: # - chart-example.local + # -- Hubble flows export. + export: + # --- Defines max file size of output file before it gets rotated. + fileMaxSizeMb: 10 + # --- Defines max number of backup/rotated files. + fileMaxBackups: 5 + # --- Static exporter configuration. + # Static exporter is bound to agent lifecycle. + static: + enabled: false + filePath: /var/run/cilium/hubble/events.log + fieldMask: [] + # - time + # - source + # - destination + # - verdict + allowList: [] + # - '{"verdict":["DROPPED","ERROR"]}' + denyList: [] + # - '{"source_pod":["kube-system/"]}' + # - '{"destination_pod":["kube-system/"]}' + # --- Dynamic exporters configuration. + # Dynamic exporters may be reconfigured without a need of agent restarts. + dynamic: + enabled: false + config: + # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. + configMapName: cilium-flowlog-config + # ---- True if helm installer should create config map. + # Switch to false if you want to self maintain the file content. + createConfigMap: true + # ---- Exporters configuration in YAML format. + content: + - name: all + fieldMask: [] + includeFilters: [] + excludeFilters: [] + filePath: "/var/run/cilium/hubble/events.log" + #- name: "test002" + # filePath: "/var/log/network/flow-log/pa/test002.log" + # fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"] + # includeFilters: + # - source_pod: ["default/"] + # event_type: + # - type: 1 + # - destination_pod: ["frontend/nginx-975996d4c-7hhgt"] + # excludeFilters: [] + # end: "2023-10-09T23:59:59-07:00" + # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" @@ -1829,8 +1912,11 @@ loadBalancer: # mode: snat # -- acceleration is the option to accelerate service handling via XDP - # e.g. native, disabled - # acceleration: disabled + # Applicable values can be: disabled (do not use XDP), native (XDP BPF + # program is run directly out of the networking driver's early receive + # path), or best-effort (use native mode XDP acceleration on devices + # that support it). + acceleration: disabled # -- dsrDispatch configures whether IP option or IPIP encapsulation is # used to pass a service IP and port to remote backend @@ -1989,14 +2075,18 @@ envoy: # -- Set Envoy upstream HTTP idle connection timeout seconds. # Does not apply to connections with pending requests. Default 60s idleTimeoutDurationSeconds: 60 + # -- Number of trusted hops regarding the x-forwarded-for and related HTTP headers for the ingress L7 policy enforcement Envoy listeners. + xffNumTrustedHopsL7PolicyIngress: 0 + # -- Number of trusted hops regarding the x-forwarded-for and related HTTP headers for the egress L7 policy enforcement Envoy listeners. + xffNumTrustedHopsL7PolicyEgress: 0 # -- Envoy container image. image: override: ~ repository: "quay.io/cilium/cilium-envoy" - tag: "v1.27.2-ab187719b71b513150f30209569695adf16ec869" + tag: "v1.33.4-1752151664-7c2edb0b44cf95f326d628b837fcdd845102ba68" pullPolicy: "IfNotPresent" - digest: "sha256:2b590be37624547d638a578a3f31278d3be53a1a2649ba888a9f15771628521e" + digest: "sha256:318eff387835ca2717baab42a84f35a83a5f9e7d519253df87269f80b9ff0171" useDigest: true # -- Additional containers added to the cilium Envoy DaemonSet. @@ -2042,8 +2132,10 @@ envoy: annotations: {} # -- Security Context for cilium-envoy pods. - podSecurityContext: {} - + podSecurityContext: + # -- AppArmorProfile options for the `cilium-agent` and init containers + appArmorProfile: + type: "Unconfined" # -- Annotations to be added to envoy pods podAnnotations: {} @@ -2112,7 +2204,20 @@ envoy: labelSelector: matchLabels: k8s-app: cilium-envoy - + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" # -- Node selector for cilium-envoy. nodeSelector: kubernetes.io/os: linux @@ -2133,12 +2238,16 @@ envoy: # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy dnsPolicy: ~ + # -- Configure Cilium Envoy Prometheus options. + # Note that some of these apply to either cilium-agent or cilium-envoy. prometheus: # -- Enable prometheus metrics for cilium-envoy enabled: true serviceMonitor: # -- Enable service monitors. # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + # Note that this setting applies to both cilium-envoy _and_ cilium-agent + # with Envoy enabled. enabled: false # -- Labels to add to ServiceMonitor cilium-envoy labels: {} @@ -2150,18 +2259,21 @@ envoy: # service monitors configured. # namespace: "" # -- Relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. relabelings: - sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: node replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. metricRelabelings: ~ # -- Serve prometheus metrics for cilium-envoy on the configured port port: "9964" # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# Deprecated without replacement in 1.15. To be removed in 1.16. remoteNodeIdentity: true # -- Enable resource quotas for priority classes used in the cluster. @@ -2242,15 +2354,6 @@ tls: # ... # -----END CERTIFICATE----- -# -- Configure the encapsulation configuration for communication between nodes. -# Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. -# Possible values: -# - disabled -# - vxlan -# - geneve -# @default -- `"vxlan"` -tunnel: "" - # -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. # Possible values: # - "" @@ -2271,6 +2374,13 @@ routingMode: "" # @default -- Port 8472 for VXLAN, Port 6081 for Geneve tunnelPort: 0 +# -- Configure what the response should be to traffic for a service without backends. +# "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". +# Possible values: +# - reject (default) +# - drop +serviceNoBackendResponse: reject + # -- Configure the underlying network MTU to overwrite auto-detected MTU. MTU: 0 @@ -2397,7 +2507,7 @@ operator: image: override: ~ repository: "quay.io/cilium/operator" - tag: "v1.15.0-pre.2" + tag: "v1.15.19" # operator-generic-digest genericDigest: "" # operator-azure-digest @@ -2600,7 +2710,9 @@ nodeinit: image: override: ~ repository: "quay.io/cilium/startup-script" - tag: "62093c5c233ea914bfa26a10ba41f8780d9b737f" + tag: "c54c7edeab7fde4da68e59acd319ab24af242c3f" + digest: "sha256:8d7b41c4ca45860254b3c19e20210462ef89479bb6331d6760c4e609d651b29c" + useDigest: true pullPolicy: "IfNotPresent" # -- The priority class to use for the nodeinit pod. @@ -2644,7 +2756,11 @@ nodeinit: # -- Labels to be added to node-init pods. podLabels: {} - + # -- Security Context for cilium-node-init pods. + podSecurityContext: + # -- AppArmorProfile options for the `cilium-node-init` and init containers + appArmorProfile: + type: "Unconfined" # -- nodeinit resource limits & requests # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: @@ -2692,7 +2808,7 @@ preflight: image: override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.15.0-pre.2" + tag: "v1.15.19" # cilium-digest digest: "" useDigest: false @@ -2809,6 +2925,12 @@ enableCriticalPriorityClass: true clustermesh: # -- Deploy clustermesh-apiserver for clustermesh useAPIServer: false + # -- The maximum number of clusters to support in a ClusterMesh. This value + # cannot be changed on running clusters, and all clusters in a ClusterMesh + # must be configured with the same value. Values > 255 will decrease the + # maximum allocatable cluster-local identities. + # Supported values are 255 and 511. + maxConnectedClusters: 255 # -- Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) annotations: {} @@ -2848,21 +2970,16 @@ clustermesh: image: override: ~ repository: "quay.io/cilium/clustermesh-apiserver" - tag: "v1.15.0-pre.2" + tag: "v1.15.19" # clustermesh-apiserver-digest digest: "" useDigest: false pullPolicy: "IfNotPresent" etcd: - # -- Clustermesh API server etcd image. - image: - override: ~ - repository: "quay.io/coreos/etcd" - tag: "v3.5.4" - digest: "sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3" - useDigest: true - pullPolicy: "IfNotPresent" + # The etcd binary is included in the clustermesh API server image, so the same image from above is reused. + # Independent override isn't supported, because clustermesh-apiserver is tested against the etcd version it is + # built with. # -- Specifies the resources for etcd container in the apiserver resources: {} @@ -2889,21 +3006,17 @@ clustermesh: # cpu: 100m # memory: 100Mi + # -- Additional arguments to `clustermesh-apiserver etcdinit`. + extraArgs: [] + + # -- Additional environment variables to `clustermesh-apiserver etcdinit`. + extraEnv: [] + kvstoremesh: # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved # from the remote clusters in the local etcd instance. enabled: false - # -- KVStoreMesh image. - image: - override: ~ - repository: "quay.io/cilium/kvstoremesh" - tag: "v1.15.0-pre.2" - # kvstoremesh-digest - digest: "" - useDigest: false - pullPolicy: "IfNotPresent" - # -- Additional KVStoreMesh arguments. extraArgs: [] @@ -2945,9 +3058,6 @@ clustermesh: # NodePort will be redirected to a local backend, regardless of whether the # destination node belongs to the local or the remote cluster. nodePort: 32379 - # -- Optional loadBalancer IP address to use with type LoadBalancer. - # loadBalancerIP: - # -- Annotations for the clustermesh-apiserver # For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 @@ -2959,6 +3069,21 @@ clustermesh: # -- The internalTrafficPolicy of service used for apiserver access. internalTrafficPolicy: + # @schema + # type: [null, string] + # @schema + # -- Configure a loadBalancerClass. + # Allows to configure the loadBalancerClass on the clustermesh-apiserver + # LB service in case the Service type is set to LoadBalancer + # (requires Kubernetes 1.24+). + loadBalancerClass: ~ + # @schema + # type: [null, string] + # @schema + # -- Configure a specific loadBalancerIP. + # Allows to configure a specific loadBalancerIP on the clustermesh-apiserver + # LB service in case the Service type is set to LoadBalancer. + loadBalancerIP: ~ # -- Number of replicas run for the clustermesh-apiserver deployment. replicas: 1 @@ -3216,7 +3341,10 @@ cgroup: # memory: 128Mi # -- Configure cgroup root where cgroup2 filesystem is mounted on the host (see also: `cgroup.autoMount`) hostRoot: /run/cilium/cgroupv2 - +# -- Configure sysctl override described in #20072. +sysctlfix: + # -- Enable the sysctl override. When enabled, the init container will mount the /proc of the host so that the `sysctlfix` utility can execute. + enabled: true # -- Configure whether to enable auto detect of terminating state for endpoints # in order to support graceful termination. enableK8sTerminatingEndpoint: true @@ -3229,6 +3357,8 @@ enableK8sTerminatingEndpoint: true agentNotReadyTaintKey: "node.cilium.io/agent-not-ready" dnsProxy: + # -- Timeout (in seconds) when closing the connection between the DNS proxy and the upstream server. If set to 0, the connection is closed immediately (with TCP RST). If set to -1, the connection is closed asynchronously in the background. + socketLingerTimeout: 10 # -- DNS response code for rejecting DNS requests, available options are '[nameError refused]'. dnsRejectResponseCode: refused # -- Allow the DNS proxy to compress responses to endpoints that are larger than 512 Bytes or the EDNS0 option, if present. @@ -3250,6 +3380,8 @@ dnsProxy: proxyPort: 0 # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. proxyResponseMaxDelay: 100ms + # -- DNS proxy operation mode (true/false, or unset to use version dependent defaults) + # enableTransparentMode: true # -- SCTP Configuration Values sctp: @@ -3289,12 +3421,14 @@ authentication: enabled: true # -- SPIRE namespace to install into namespace: cilium-spire + # -- SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. + existingNamespace: false # -- init container image of SPIRE agent and server initImage: override: ~ repository: "docker.io/library/busybox" - tag: "1.35.0" - digest: "sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b" + tag: "1.36.1" + digest: "sha256:7edf5efe6b86dbf01ccc3c76b32a37a8e23b84e6bad81ce8ae8c221fa456fda8" useDigest: true pullPolicy: "IfNotPresent" # SPIRE agent configuration @@ -3303,8 +3437,8 @@ authentication: image: override: ~ repository: "ghcr.io/spiffe/spire-agent" - tag: "1.6.3" - digest: "sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823" + tag: "1.8.5" + digest: "sha256:99405637647968245ff9fe215f8bd2bd0ea9807be9725f8bf19fe1b21471e52b" useDigest: true pullPolicy: "IfNotPresent" # -- SPIRE agent service account @@ -3318,15 +3452,41 @@ authentication: # -- SPIRE Workload Attestor kubelet verification. skipKubeletVerification: true # -- SPIRE agent tolerations configuration + # By default it follows the same tolerations as the agent itself + # to allow the Cilium agent on this node to connect to SPIRE. # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - tolerations: [] + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" + # -- SPIRE agent affinity configuration + affinity: {} + # -- SPIRE agent nodeSelector configuration + # ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: {} + # -- Security context to be added to spire agent pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire agent containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} server: # -- SPIRE server image image: override: ~ repository: "ghcr.io/spiffe/spire-server" - tag: "1.6.3" - digest: "sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f" + tag: "1.8.5" + digest: "sha256:28269265882048dcf0fed32fe47663cd98613727210b8d1a55618826f9bf5428" useDigest: true pullPolicy: "IfNotPresent" # -- SPIRE server service account diff --git a/internal/constellation/helm/charts/cilium/values.yaml.tmpl b/internal/constellation/helm/charts/cilium/values.yaml.tmpl index bb22548c8..7fbed1be7 100644 --- a/internal/constellation/helm/charts/cilium/values.yaml.tmpl +++ b/internal/constellation/helm/charts/cilium/values.yaml.tmpl @@ -44,11 +44,13 @@ k8sServicePort: "" # rate limit, the agent and operator will start to throttle requests by delaying # them until there is budget or the request times out. k8sClientRateLimit: - # -- The sustained request rate in requests per second. - qps: 5 - # -- The burst request rate in requests per second. + # -- (int) The sustained request rate in requests per second. + # @default -- 5 for k8s up to 1.26. 10 for k8s version 1.27+ + qps: + # -- (int) The burst request rate in requests per second. # The rate limiter will allow short bursts with a higher rate. - burst: 10 + # @default -- 10 for k8s up to 1.26. 20 for k8s version 1.27+ + burst: cluster: # -- Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. @@ -213,8 +215,10 @@ extraConfig: {} annotations: {} # -- Security Context for cilium-agent pods. -podSecurityContext: {} - +podSecurityContext: + # -- AppArmorProfile options for the `cilium-agent` and init containers + appArmorProfile: + type: "Unconfined" # -- Annotations to be added to agent pods podAnnotations: {} @@ -231,6 +235,9 @@ resources: {} # cpu: 100m # memory: 512Mi +# -- resources & limits for the agent init containers +initResources: {} + securityContext: # -- User to run the pod with # runAsUser: 0 @@ -409,9 +416,9 @@ bgpControlPlane: # -- SecretsNamespace is the namespace which BGP support will retrieve secrets from. secretsNamespace: # -- Create secrets namespace for BGP secrets. - create: true + create: false # -- The name of the secret namespace to which Cilium agents are given read access - name: cilium-bgp-secrets + name: kube-system pmtuDiscovery: # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to @@ -460,7 +467,17 @@ bpf: # @default -- `524288` neighMax: ~ + # @schema + # type: [null, integer] + # @schema + # @default -- `16384` + # -- (int) Configures the maximum number of entries for the node table. + nodeMapMax: ~ + # -- Configure the maximum number of entries in endpoint policy map (per endpoint). + # @schema + # type: [null, integer] + # @schema policyMapMax: 16384 # -- (float64) Configure auto-sizing for all BPF maps based on available memory. @@ -593,6 +610,12 @@ cni: # inside the agent pod. hostConfDirMountPath: /host/etc/cni/net.d + # -- Specifies the resources for the cni initContainer + resources: + requests: + cpu: 100m + memory: 10Mi + # -- (string) Configure how frequently garbage collection should occur for the datapath # connection tracking table. # @default -- `"0s"` @@ -670,13 +693,6 @@ enableRuntimeDeviceDetection: false # -- Limit iptables-based egress masquerading to interface selector. # egressMasqueradeInterfaces: "" -# -- Whether to enable CNP status updates. -enableCnpStatusUpdates: false - -# -- Configures the use of the KVStore to optimize Kubernetes event handling by -# mirroring it into the KVstore for reduced overhead in large clusters. -enableK8sEventHandover: false - # -- Enable setting identity mark for local traffic. # enableIdentityMark: true @@ -721,8 +737,7 @@ ingressController: # -- Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. enableProxyProtocol: false - # -- IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate - # from Ingress to the Load Balancer service + # -- IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] # -- Default secret namespace for ingresses without .spec.tls[].secretName set. @@ -800,17 +815,21 @@ encryption: # This option is only effective when encryption.type is set to "wireguard". nodeEncryption: false - # -- Configure the WireGuard Pod2Pod strict mode. + # -- Configure the WireGuard strict mode. strictMode: - # -- Enable WireGuard Pod2Pod strict mode. + # -- Enable WireGuard strict mode. enabled: false - # -- CIDR for the WireGuard Pod2Pod strict mode. - cidr: "" + # -- podCIDRList for the WireGuard strict mode. + podCIDRList: [] + + # -- nodeCIDRList for the WireGuard strict mode. + nodeCIDRList: [] # -- Allow dynamic lookup of remote node identities. # This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. - allowRemoteNodeIdentities: false + # This is also required when control-plane nodes are exempted from node-to-node encryption. + allowRemoteNodeIdentities: true ipsec: # -- Name of the key file inside the Kubernetes secret configured via secretName. @@ -1079,6 +1098,19 @@ hubble: # --set hubble.redact.enabled="true" # --set hubble.redact.http.urlQuery="true" urlQuery: false + # -- Enables redacting user info, e.g., password when basic auth is used. + # Example: + # + # redact: + # enabled: true + # http: + # userInfo: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.userInfo="true" + userInfo: true headers: # -- List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. # Example: @@ -1122,7 +1154,7 @@ hubble: # # --set hubble.redact.enabled="true" # --set hubble.redact.kafka.apiKey="true" - apiKey: false + apiKey: true # -- An additional address for Hubble to listen to. # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that @@ -1276,6 +1308,12 @@ hubble: rollingUpdate: maxUnavailable: 1 + # -- Additional hubble-relay volumes. + extraVolumes: [] + + # -- Additional hubble-relay volumeMounts. + extraVolumeMounts: [] + # -- hubble-relay pod security context podSecurityContext: fsGroup: 65532 @@ -1590,6 +1628,55 @@ hubble: # hosts: # - chart-example.local + # -- Hubble flows export. + export: + # --- Defines max file size of output file before it gets rotated. + fileMaxSizeMb: 10 + # --- Defines max number of backup/rotated files. + fileMaxBackups: 5 + # --- Static exporter configuration. + # Static exporter is bound to agent lifecycle. + static: + enabled: false + filePath: /var/run/cilium/hubble/events.log + fieldMask: [] + # - time + # - source + # - destination + # - verdict + allowList: [] + # - '{"verdict":["DROPPED","ERROR"]}' + denyList: [] + # - '{"source_pod":["kube-system/"]}' + # - '{"destination_pod":["kube-system/"]}' + # --- Dynamic exporters configuration. + # Dynamic exporters may be reconfigured without a need of agent restarts. + dynamic: + enabled: false + config: + # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. + configMapName: cilium-flowlog-config + # ---- True if helm installer should create config map. + # Switch to false if you want to self maintain the file content. + createConfigMap: true + # ---- Exporters configuration in YAML format. + content: + - name: all + fieldMask: [] + includeFilters: [] + excludeFilters: [] + filePath: "/var/run/cilium/hubble/events.log" + #- name: "test002" + # filePath: "/var/log/network/flow-log/pa/test002.log" + # fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"] + # includeFilters: + # - source_pod: ["default/"] + # event_type: + # - type: 1 + # - destination_pod: ["frontend/nginx-975996d4c-7hhgt"] + # excludeFilters: [] + # end: "2023-10-09T23:59:59-07:00" + # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" @@ -1822,8 +1909,11 @@ loadBalancer: # mode: snat # -- acceleration is the option to accelerate service handling via XDP - # e.g. native, disabled - # acceleration: disabled + # Applicable values can be: disabled (do not use XDP), native (XDP BPF + # program is run directly out of the networking driver's early receive + # path), or best-effort (use native mode XDP acceleration on devices + # that support it). + acceleration: disabled # -- dsrDispatch configures whether IP option or IPIP encapsulation is # used to pass a service IP and port to remote backend @@ -1982,6 +2072,10 @@ envoy: # -- Set Envoy upstream HTTP idle connection timeout seconds. # Does not apply to connections with pending requests. Default 60s idleTimeoutDurationSeconds: 60 + # -- Number of trusted hops regarding the x-forwarded-for and related HTTP headers for the ingress L7 policy enforcement Envoy listeners. + xffNumTrustedHopsL7PolicyIngress: 0 + # -- Number of trusted hops regarding the x-forwarded-for and related HTTP headers for the egress L7 policy enforcement Envoy listeners. + xffNumTrustedHopsL7PolicyEgress: 0 # -- Envoy container image. image: @@ -2035,8 +2129,10 @@ envoy: annotations: {} # -- Security Context for cilium-envoy pods. - podSecurityContext: {} - + podSecurityContext: + # -- AppArmorProfile options for the `cilium-agent` and init containers + appArmorProfile: + type: "Unconfined" # -- Annotations to be added to envoy pods podAnnotations: {} @@ -2105,7 +2201,20 @@ envoy: labelSelector: matchLabels: k8s-app: cilium-envoy - + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" # -- Node selector for cilium-envoy. nodeSelector: kubernetes.io/os: linux @@ -2126,12 +2235,16 @@ envoy: # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy dnsPolicy: ~ + # -- Configure Cilium Envoy Prometheus options. + # Note that some of these apply to either cilium-agent or cilium-envoy. prometheus: # -- Enable prometheus metrics for cilium-envoy enabled: true serviceMonitor: # -- Enable service monitors. # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + # Note that this setting applies to both cilium-envoy _and_ cilium-agent + # with Envoy enabled. enabled: false # -- Labels to add to ServiceMonitor cilium-envoy labels: {} @@ -2143,18 +2256,21 @@ envoy: # service monitors configured. # namespace: "" # -- Relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. relabelings: - sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: node replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. metricRelabelings: ~ # -- Serve prometheus metrics for cilium-envoy on the configured port port: "9964" # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# Deprecated without replacement in 1.15. To be removed in 1.16. remoteNodeIdentity: true # -- Enable resource quotas for priority classes used in the cluster. @@ -2235,15 +2351,6 @@ tls: # ... # -----END CERTIFICATE----- -# -- Configure the encapsulation configuration for communication between nodes. -# Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. -# Possible values: -# - disabled -# - vxlan -# - geneve -# @default -- `"vxlan"` -tunnel: "" - # -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. # Possible values: # - "" @@ -2264,6 +2371,13 @@ routingMode: "" # @default -- Port 8472 for VXLAN, Port 6081 for Geneve tunnelPort: 0 +# -- Configure what the response should be to traffic for a service without backends. +# "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". +# Possible values: +# - reject (default) +# - drop +serviceNoBackendResponse: reject + # -- Configure the underlying network MTU to overwrite auto-detected MTU. MTU: 0 @@ -2594,6 +2708,8 @@ nodeinit: override: ~ repository: "${CILIUM_NODEINIT_REPO}" tag: "${CILIUM_NODEINIT_VERSION}" + digest: "${CILIUM_NODEINIT_DIGEST}" + useDigest: true pullPolicy: "${PULL_POLICY}" # -- The priority class to use for the nodeinit pod. @@ -2637,7 +2753,11 @@ nodeinit: # -- Labels to be added to node-init pods. podLabels: {} - + # -- Security Context for cilium-node-init pods. + podSecurityContext: + # -- AppArmorProfile options for the `cilium-node-init` and init containers + appArmorProfile: + type: "Unconfined" # -- nodeinit resource limits & requests # ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: @@ -2802,6 +2922,12 @@ enableCriticalPriorityClass: true clustermesh: # -- Deploy clustermesh-apiserver for clustermesh useAPIServer: false + # -- The maximum number of clusters to support in a ClusterMesh. This value + # cannot be changed on running clusters, and all clusters in a ClusterMesh + # must be configured with the same value. Values > 255 will decrease the + # maximum allocatable cluster-local identities. + # Supported values are 255 and 511. + maxConnectedClusters: 255 # -- Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) annotations: {} @@ -2848,14 +2974,9 @@ clustermesh: pullPolicy: "${PULL_POLICY}" etcd: - # -- Clustermesh API server etcd image. - image: - override: ~ - repository: "${ETCD_REPO}" - tag: "${ETCD_VERSION}" - digest: "${ETCD_DIGEST}" - useDigest: true - pullPolicy: "${PULL_POLICY}" + # The etcd binary is included in the clustermesh API server image, so the same image from above is reused. + # Independent override isn't supported, because clustermesh-apiserver is tested against the etcd version it is + # built with. # -- Specifies the resources for etcd container in the apiserver resources: {} @@ -2882,21 +3003,17 @@ clustermesh: # cpu: 100m # memory: 100Mi + # -- Additional arguments to `clustermesh-apiserver etcdinit`. + extraArgs: [] + + # -- Additional environment variables to `clustermesh-apiserver etcdinit`. + extraEnv: [] + kvstoremesh: # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved # from the remote clusters in the local etcd instance. enabled: false - # -- KVStoreMesh image. - image: - override: ~ - repository: "${KVSTOREMESH_REPO}" - tag: "${CILIUM_VERSION}" - # kvstoremesh-digest - digest: ${KVSTOREMESH_DIGEST} - useDigest: ${USE_DIGESTS} - pullPolicy: "${PULL_POLICY}" - # -- Additional KVStoreMesh arguments. extraArgs: [] @@ -2938,9 +3055,6 @@ clustermesh: # NodePort will be redirected to a local backend, regardless of whether the # destination node belongs to the local or the remote cluster. nodePort: 32379 - # -- Optional loadBalancer IP address to use with type LoadBalancer. - # loadBalancerIP: - # -- Annotations for the clustermesh-apiserver # For GKE LoadBalancer, use annotation cloud.google.com/load-balancer-type: "Internal" # For EKS LoadBalancer, use annotation service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 @@ -2952,6 +3066,21 @@ clustermesh: # -- The internalTrafficPolicy of service used for apiserver access. internalTrafficPolicy: + # @schema + # type: [null, string] + # @schema + # -- Configure a loadBalancerClass. + # Allows to configure the loadBalancerClass on the clustermesh-apiserver + # LB service in case the Service type is set to LoadBalancer + # (requires Kubernetes 1.24+). + loadBalancerClass: ~ + # @schema + # type: [null, string] + # @schema + # -- Configure a specific loadBalancerIP. + # Allows to configure a specific loadBalancerIP on the clustermesh-apiserver + # LB service in case the Service type is set to LoadBalancer. + loadBalancerIP: ~ # -- Number of replicas run for the clustermesh-apiserver deployment. replicas: 1 @@ -3209,7 +3338,10 @@ cgroup: # memory: 128Mi # -- Configure cgroup root where cgroup2 filesystem is mounted on the host (see also: `cgroup.autoMount`) hostRoot: /run/cilium/cgroupv2 - +# -- Configure sysctl override described in #20072. +sysctlfix: + # -- Enable the sysctl override. When enabled, the init container will mount the /proc of the host so that the `sysctlfix` utility can execute. + enabled: true # -- Configure whether to enable auto detect of terminating state for endpoints # in order to support graceful termination. enableK8sTerminatingEndpoint: true @@ -3222,6 +3354,8 @@ enableK8sTerminatingEndpoint: true agentNotReadyTaintKey: "node.cilium.io/agent-not-ready" dnsProxy: + # -- Timeout (in seconds) when closing the connection between the DNS proxy and the upstream server. If set to 0, the connection is closed immediately (with TCP RST). If set to -1, the connection is closed asynchronously in the background. + socketLingerTimeout: 10 # -- DNS response code for rejecting DNS requests, available options are '[nameError refused]'. dnsRejectResponseCode: refused # -- Allow the DNS proxy to compress responses to endpoints that are larger than 512 Bytes or the EDNS0 option, if present. @@ -3243,6 +3377,8 @@ dnsProxy: proxyPort: 0 # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. proxyResponseMaxDelay: 100ms + # -- DNS proxy operation mode (true/false, or unset to use version dependent defaults) + # enableTransparentMode: true # -- SCTP Configuration Values sctp: @@ -3282,6 +3418,8 @@ authentication: enabled: true # -- SPIRE namespace to install into namespace: cilium-spire + # -- SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. + existingNamespace: false # -- init container image of SPIRE agent and server initImage: override: ~ @@ -3311,8 +3449,34 @@ authentication: # -- SPIRE Workload Attestor kubelet verification. skipKubeletVerification: true # -- SPIRE agent tolerations configuration + # By default it follows the same tolerations as the agent itself + # to allow the Cilium agent on this node to connect to SPIRE. # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - tolerations: [] + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" + # -- SPIRE agent affinity configuration + affinity: {} + # -- SPIRE agent nodeSelector configuration + # ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: {} + # -- Security context to be added to spire agent pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire agent containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} server: # -- SPIRE server image image: diff --git a/internal/constellation/helm/charts/coredns/Chart.yaml b/internal/constellation/helm/charts/coredns/Chart.yaml new file mode 100644 index 000000000..bd531acd7 --- /dev/null +++ b/internal/constellation/helm/charts/coredns/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: kube-dns +version: 0.0.0 diff --git a/internal/constellation/helm/charts/coredns/templates/clusterrole.yaml b/internal/constellation/helm/charts/coredns/templates/clusterrole.yaml new file mode 100644 index 000000000..13d284c3c --- /dev/null +++ b/internal/constellation/helm/charts/coredns/templates/clusterrole.yaml @@ -0,0 +1,23 @@ + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch diff --git a/internal/constellation/helm/charts/coredns/templates/clusterrolebinding.yaml b/internal/constellation/helm/charts/coredns/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..ab35291a1 --- /dev/null +++ b/internal/constellation/helm/charts/coredns/templates/clusterrolebinding.yaml @@ -0,0 +1,13 @@ + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system diff --git a/internal/constellation/helm/charts/coredns/templates/configmap.yaml b/internal/constellation/helm/charts/coredns/templates/configmap.yaml new file mode 100644 index 000000000..03f06d623 --- /dev/null +++ b/internal/constellation/helm/charts/coredns/templates/configmap.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes {{ .Values.dnsDomain }} in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 { + disable success {{ .Values.dnsDomain }} + disable denial {{ .Values.dnsDomain }} + } + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + creationTimestamp: null + name: edg-coredns + namespace: kube-system diff --git a/internal/constellation/helm/charts/coredns/templates/deployment.yaml b/internal/constellation/helm/charts/coredns/templates/deployment.yaml new file mode 100644 index 000000000..b7fd735df --- /dev/null +++ b/internal/constellation/helm/charts/coredns/templates/deployment.yaml @@ -0,0 +1,109 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + name: coredns + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: '{{ .Values.image }}' + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 10 + volumes: + - configMap: + items: + - key: Corefile + path: Corefile + name: edg-coredns + name: config-volume +status: {} diff --git a/internal/constellation/helm/charts/coredns/templates/service.yaml b/internal/constellation/helm/charts/coredns/templates/service.yaml new file mode 100644 index 000000000..ba243aa19 --- /dev/null +++ b/internal/constellation/helm/charts/coredns/templates/service.yaml @@ -0,0 +1,33 @@ + +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + # Without this resourceVersion value, an update of the Service between versions will yield: + # Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update + resourceVersion: "0" +spec: + clusterIP: "{{ .Values.clusterIP }}" + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + - name: metrics + port: 9153 + protocol: TCP + targetPort: 9153 + selector: + k8s-app: kube-dns diff --git a/internal/constellation/helm/charts/coredns/templates/serviceaccount.yaml b/internal/constellation/helm/charts/coredns/templates/serviceaccount.yaml new file mode 100644 index 000000000..937b99fa5 --- /dev/null +++ b/internal/constellation/helm/charts/coredns/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system diff --git a/internal/constellation/helm/charts/coredns/values.yaml b/internal/constellation/helm/charts/coredns/values.yaml new file mode 100644 index 000000000..c740ed71c --- /dev/null +++ b/internal/constellation/helm/charts/coredns/values.yaml @@ -0,0 +1,3 @@ +clusterIP: 10.96.0.10 +dnsDomain: cluster.local +image: registry.k8s.io/coredns/coredns:v1.12.0@sha256:40384aa1f5ea6bfdc77997d243aec73da05f27aed0c5e9d65bfa98933c519d97 diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/Chart.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/Chart.yaml index 87a6d0c4e..9980be91a 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/constellation-services/Chart.yaml @@ -45,25 +45,7 @@ dependencies: - GCP - OpenStack - QEMU - - name: konnectivity - version: 0.0.0 - tags: - - AWS - - Azure - - GCP - - OpenStack - - QEMU - name: gcp-guest-agent version: 0.0.0 tags: - GCP - - name: yawol-config - version: 0.0.0 - condition: openstack.deployYawolLoadBalancer - tags: - - OpenStack - - name: yawol-controller - version: 0.0.0 - condition: openstack.deployYawolLoadBalancer - tags: - - OpenStack diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml new file mode 100644 index 000000000..ac479a068 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns-pdb + namespace: "kube-system" +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml index 06a971465..220c3efc6 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml +++ b/internal/constellation/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml @@ -5,5 +5,11 @@ metadata: name: gceconf namespace: {{ .Release.Namespace }} data: - gce.conf: "[global]\nproject-id = {{.Values.GCP.projectID }}\nuse-metadata-server = true\nnode-tags = constellation-{{ .Values.GCP.uid }}\nregional = true\n" + gce.conf: | + [global] + project-id = {{.Values.GCP.projectID }} + use-metadata-server = true + node-tags = constellation-{{ .Values.GCP.uid }} + regional = true + token-url = nil # This forces use of GOOGLE_APPLICATION_CREDENTIALS. {{- end -}} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml index fe6460d4a..f14515244 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/charts/edgeless/constellation-services/charts/join-service/templates/daemonset.yaml @@ -40,6 +40,9 @@ spec: - --cloud-provider={{ .Values.csp }} - --key-service-endpoint=key-service.{{ .Release.Namespace }}:{{ .Values.global.keyServicePort }} - --attestation-variant={{ .Values.attestationVariant }} + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json volumeMounts: - mountPath: {{ .Values.global.serviceBasePath | quote }} name: config @@ -47,6 +50,14 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /var/run/state/ssh + name: ssh ports: - containerPort: {{ .Values.joinServicePort }} name: tcp @@ -54,6 +65,10 @@ spec: securityContext: privileged: true volumes: + - name: gcekey + secret: + secretName: gcekey + optional: true - name: config projected: sources: @@ -64,4 +79,10 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: ssh + hostPath: + path: /var/run/state/ssh updateStrategy: {} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/Chart.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/Chart.yaml deleted file mode 100644 index 010e5d071..000000000 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: konnectivity -description: A chart to deploy konnectivity for Constellation -type: application -version: 0.0.0 diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml deleted file mode 100644 index f189cb6a3..000000000 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: system:konnectivity-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:konnectivity-server diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/daemonset.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/daemonset.yaml deleted file mode 100644 index d195e8036..000000000 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/daemonset.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: konnectivity-agent - name: konnectivity-agent - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - k8s-app: konnectivity-agent - template: - metadata: - labels: - k8s-app: konnectivity-agent - spec: - containers: - - args: - - --logtostderr=true - - --proxy-server-host={{ .Values.loadBalancerIP }} - - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --proxy-server-port=8132 - - --admin-server-port=8133 - - --health-server-port={{ .Values.healthServerPort }} - - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token - - --agent-identifiers=host=$(HOST_IP) - - --sync-forever=true - - --keepalive-time=60m - - --sync-interval=5s - - --sync-interval-cap=30s - - --probe-interval=5s - - --v=3 - command: - - /proxy-agent - env: - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - image: {{ .Values.image | quote }} - livenessProbe: - httpGet: - path: /healthz - port: {{ .Values.healthServerPort }} - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: konnectivity-agent - resources: {} - volumeMounts: - - mountPath: /var/run/secrets/tokens - name: konnectivity-agent-token - readOnly: true - priorityClassName: system-cluster-critical - serviceAccountName: konnectivity-agent - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - name: konnectivity-agent-token - projected: - sources: - - serviceAccountToken: - audience: system:konnectivity-server - path: konnectivity-agent-token - updateStrategy: {} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/serviceaccount.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/serviceaccount.yaml deleted file mode 100644 index d48b23430..000000000 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: konnectivity-agent - namespace: {{ .Release.Namespace }} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/values.schema.json b/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/values.schema.json deleted file mode 100644 index 50f9c0de3..000000000 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/values.schema.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema#", - "properties": { - "image": { - "description": "Container image to use for the spawned pods.", - "type": "string", - "examples": ["us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent:v0.0.33@sha256:48f2a4ec3e10553a81b8dd1c6fa5fe4bcc9617f78e71c1ca89c6921335e2d7da"] - }, - "loadBalancerIP": { - "description": "IP of the loadbalancer serving the control plane.", - "type": "string", - "examples": ["10.4.0.1"] - } - }, - "required": [ - "image", - "loadBalancerIP" - ], - "title": "Values", - "type": "object" -} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/values.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/values.yaml deleted file mode 100644 index 61ffc1a85..000000000 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/values.yaml +++ /dev/null @@ -1 +0,0 @@ -healthServerPort: 8134 diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/values.yaml b/internal/constellation/helm/charts/edgeless/constellation-services/values.yaml index 05fd7c910..6f97e8e3e 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/values.yaml +++ b/internal/constellation/helm/charts/edgeless/constellation-services/values.yaml @@ -8,10 +8,6 @@ global: # Name of the ConfigMap that holds configs that should not be modified by the user. internalCMName: internal-config -# OpenStack specific configuration -openstack: - deployYawolLoadBalancer: false - # Set one of the tags to true to indicate which CSP you are deploying to. tags: AWS: false diff --git a/internal/constellation/helm/charts/edgeless/csi/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/Chart.yaml index c947d8b8a..5301e51db 100644 --- a/internal/constellation/helm/charts/edgeless/csi/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/Chart.yaml @@ -9,11 +9,11 @@ dependencies: - name: snapshot-crds version: 6.2.2 - name: aws-csi-driver - version: 1.1.0 + version: 1.2.0 tags: - AWS - name: azuredisk-csi-driver - version: v1.3.0 + version: v1.4.0 tags: - Azure - name: cinder-config @@ -21,10 +21,10 @@ dependencies: tags: - OpenStack - name: gcp-compute-persistent-disk-csi-driver - version: 1.3.0 + version: 1.4.0 tags: - GCP - name: openstack-cinder-csi - version: 1.0.0 + version: 1.0.2 tags: - OpenStack diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md index bc6aa0036..3daf61e5e 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/CHANGELOG.md @@ -1,4 +1,120 @@ # Helm chart +## v2.30.0 +* Bump driver version to `v1.30.0` +* Update voluemessnapshotcontents/status RBAC ([#1991](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1991), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Upgrade dependencies ([#2016](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2016), [@torredil](https://github.com/torredil)) + +## v2.29.1 +* Bump driver version to `v1.29.1` +* Remove `--reuse-values` deprecation warning + +## v2.29.0 +### Urgent Upgrade Notes +*(No, really, you MUST read this before you upgrade)* + +The EBS CSI Driver Helm chart no longer supports upgrading with `--reuse-values`. This chart will not test for `--reuse-values` compatibility and upgrading with `--reuse-values` will likely fail. Users of `--reuse-values` are strongly encouraged to migrate to `--reset-then-reuse-values`. + +For more information see [the deprecation announcement](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864). + +### Other Changes +* Bump driver version to `v1.29.0` and sidecars to latest versions +* Add helm-tester enabled flag ([#1954](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1954), [@nunodomingues-td](https://github.com/nunodomingues-td)) + +## v2.28.1 +* Add `reservedVolumeAttachments` that overrides heuristic-determined reserved attachments via `--reserved-volume-attachments` CLI option from [PR #1919](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1919) through Helm ([#1939](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1939), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add `additionalArgs` parameter to node daemonSet ([#1939](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1939), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.28.0 +### Urgent Upgrade Notes +*(No, really, you MUST read this before you upgrade)* + +This is the last minor version of the EBS CSI Driver Helm chart to support upgrading with `--reuse-values`. Future versions of the chart (starting with `v2.29.0`) will not test for `--reuse-values` compatibility and upgrading with `--reuse-values` will likely fail. Users of `--reuse-values` are strongly encouraged to migrate to `--reset-then-reuse-values`. + +For more information see [the deprecation announcement](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864). + +### Other Changes +* Bump driver version to `v1.28.0` and sidecars to latest versions +* Add labels to leases role used by EBS CSI controller ([#1914](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1914), [@cHiv0rz](https://github.com/cHiv0rz)) +* Enforce `linux` and `amd64` node affinity for helm tester pod ([#1922](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1922), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add configuration for `DaemonSet` annotations ([#1923](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1923), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Incorporate KubeLinter recommended best practices for chart tester pod ([#1924](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1924), [@torredil](https://github.com/torredil)) +* Add configuration for chart tester pod image ([#1928](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1928), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.27.0 +* Bump driver version to `v1.27.0` +* Add parameters for tuning revisionHistoryLimit and emptyDir volumes ([#1840](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1840), [@bodgit](https://github.com/bodgit)) + +## v2.26.1 +* Bump driver version to `v1.26.1` +* Bump sidecar container versions to fix [restart bug in external attacher, provisioner, resizer, snapshotter, and node-driver-registrar](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1875) ([#1886](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1886), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.26.0 +* Bump driver version to `v1.26.0` +* Bump sidecar container versions ([#1867](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1867), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add warning about --reuse-values deprecation to NOTES.txt ([#1865](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1865), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.25.0 +* Bump driver version to `v1.25.0` +* Update default sidecar timeout values ([#1824](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1824), [@torredil](https://github.com/torredil)) +* Increase default QPS and worker threads of sidecars ([#1834](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1834), [@ConnorJC3](https://github.com/ConnorJC3)) +* Node-driver-registrar sidecar fixes ([#1815](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1815), [@jukie](https://github.com/jukie)) +* Suggest eks.amazonaws.com/role-arn in values.yaml if EKS IAM for SA is used ([#1804](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1804), [@tporeba](https://github.com/tporeba)) + +## v2.24.1 +* Bump driver version to `v1.24.1` +* Upgrade sidecar images + +## v2.24.0 +* Bump driver version to `v1.24.0` +* Add additionalClusterRoleRules to sidecar chart templates. ([#1757](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1757), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Allow passing template value for clusterName ([#1753](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1753), [@monicastanciu](https://github.com/monicastanciu)) +* Make hostNetwork configurable for daemonset ([#1716](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1716), [@bseenu](https://github.com/bseenu)) +* Add labels to volumesnapshotclass ([#1754](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1754), [@fad3t](https://github.com/fad3t)) +* Update default API version for PodDisruptionBudget ([#1751](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1751), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.23.2 +* Bump driver version to `v1.23.2` +* Upgrade sidecar images + +## v2.23.1 +* Bump driver version to `v1.23.1` + +## v2.23.0 +* Add `node.enableLinux` parameter ([#1732](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1732), [@monicastanciu](https://github.com/monicastanciu)) +* Additional Node DaemonSets bug fixes ([#1739](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1739), [@monicastanciu](https://github.com/monicastanciu)) +* Additional DaemonSets feature ([#1722](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1722), [@ConnorJC3](https://github.com/ConnorJC3)) +* Add doc of chart value additionalArgs ([#1697](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1697), [@zitudu](https://github.com/zitudu)) + +## v2.22.1 +* Bump driver version to `v1.22.1` + +## v2.22.0 +* Default PodDisruptionBudget to policy/v1 ([#1707](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1707), [@iNoahNothing](https://github.com/iNoahNothing)) + +## v2.21.0 +* Bump driver version to `v1.21.0` +* Enable additional volume mounts on node pods ([#1670](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1670), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Enable customization of aws-secret name and keys in Helm Chart ([#1668](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1668), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* The sidecars have been updated. The new versions are: + - csi-snapshotter: `v6.2.2` + +## v2.20.0 +* Bump driver version to `v1.20.0` +* Enable leader election in csi-resizer sidecar ([#1606](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1606), [@rdpsin](https://github.com/rdpsin)) +* Namespace-scoped leases permissions ([#1614](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1614), [@torredil](https://github.com/torredil)) +* Add additionalArgs parameter for sidecars ([#1627](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1627), [@ConnorJC3](https://github.com/ConnorJC3)) +* Avoid generating manifests with empty envFrom fields ([#1630](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1630), [@mvgmb](https://github.com/mvgmb)) +* Allow to set automountServiceAccountToken in ServiceAccount ([#1619](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1619), [@kahirokunn](https://github.com/kahirokunn)) + +## v2.19.0 +* Bump driver version to `v1.19.0` +* The sidecars have been updated. The new versions are: + - csi-provisioner: `v3.5.0` + - csi-attacher: `v4.3.0` + - livenessprobe: `v2.10.0` + - csi-resizer: `v1.8.0` + - node-driver-registrar: `v2.8.0` +* Remove CPU limits ([#1596](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1596), [@torredil](https://github.com/torredil)) ## v2.18.0 ### Urgent Upgrade Notes diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml index fc4e85297..c439f3ef6 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -version: 1.1.0 -appVersion: "1.1.0" +version: 1.2.0 +appVersion: "1.2.0" description: AWS Container Storage Interface (CSI) Storage Plugin with on-node encryption support name: aws-csi-driver kubeVersion: ">=1.17.0-0" diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt index 5d79084ec..cb3e6cecf 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/NOTES.txt @@ -2,4 +2,4 @@ To verify that aws-ebs-csi-driver has started, run: kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. \ No newline at end of file +NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl new file mode 100644 index 000000000..ab17f71e5 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node-windows.tpl @@ -0,0 +1,262 @@ +{{- define "node-windows" }} +{{- if .Values.node.enableWindows }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ printf "%s-windows" .NodeName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{ toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: windows + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.node.windowsHostProcess }} + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.node.windowsHostProcess }} + command: + - "aws-ebs-csi-driver.exe" + {{- end }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- if .Values.node.otelTracing }} + - --enable-otel-tracing=true + {{- end}} + {{- if .Values.node.windowsHostProcess }} + - --windows-host-process=true + {{- end }} + env: + - name: CSI_ENDPOINT + {{- if .Values.node.windowsHostProcess }} + value: unix://C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + value: unix:/csi/csi.sock + {{- end }} + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: C:\var\lib\kubelet + mountPropagation: "None" + - name: plugin-dir + mountPath: C:\csi + {{- if not .Values.node.windowsHostProcess }} + - name: csi-proxy-disk-pipe + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe + mountPath: \\.\pipe\csi-proxy-volume-v1 + - name: csi-proxy-filesystem-pipe + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if not .Values.node.windowsHostProcess }} + securityContext: + windowsOptions: + runAsUserName: "ContainerAdministrator" + {{- end }} + lifecycle: + preStop: + exec: + command: ["/bin/aws-ebs-csi-driver", "pre-stop-hook"] + - name: node-driver-registrar + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + {{- if .Values.node.windowsHostProcess }} + command: + - "csi-node-driver-registrar.exe" + {{- end }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + {{- if .Values.node.windowsHostProcess }} + - --plugin-registration-path=$(PLUGIN_REG_DIR) + {{- end }} + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + env: + - name: ADDRESS + {{- if .Values.node.windowsHostProcess }} + value: unix://C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + value: unix:/csi/csi.sock + {{- end }} + - name: DRIVER_REG_SOCK_PATH + {{- if .Values.node.windowsHostProcess }} + value: C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock + {{- end }} + {{- if .Values.node.windowsHostProcess }} + - name: PLUGIN_REG_DIR + value: C:\\var\\lib\\kubelet\\plugins_registry\\ + {{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + periodSeconds: 90 + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + - name: probe-dir + mountPath: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + {{- if .Values.node.windowsHostProcess }} + command: + - "livenessprobe.exe" + {{- end }} + args: + {{- if .Values.node.windowsHostProcess }} + - --csi-address=unix://C:\\var\\lib\\kubelet\\plugins\\ebs.csi.aws.com\\csi.sock + {{- else }} + - --csi-address=unix:/csi/csi.sock + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry + type: Directory + {{- if not .Values.node.windowsHostProcess }} + - name: csi-proxy-disk-pipe + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + - name: csi-proxy-volume-pipe + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + - name: csi-proxy-filesystem-pipe + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" + {{- end }} + - name: probe-dir + {{- if .Values.node.probeDirVolume }} + {{- toYaml .Values.node.probeDirVolume | nindent 10 }} + {{- else }} + emptyDir: {} + {{- end }} +{{- end }} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl new file mode 100644 index 000000000..4591f7efe --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/_node.tpl @@ -0,0 +1,250 @@ +{{- define "node" }} +{{- if or (eq (default true .Values.node.enableLinux) true) }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .NodeName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.daemonSetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }} + {{- end }} + selector: + matchLabels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: {{ .NodeName }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + - key: "ebs.csi.aws.com/agent-not-ready" + operator: "Exists" + {{- end }} + hostNetwork: {{ .Values.node.hostNetwork }} + {{- with .Values.node.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.reservedVolumeAttachments }} + - --reserved-volume-attachments={{ . }} + {{- end }} + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- if .Values.node.otelTracing }} + - --enable-otel-tracing=true + {{- end}} + {{- range .Values.node.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: {{ .Values.node.kubeletPath }} + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + - name: cryptsetup + mountPath: /run/cryptsetup + {{- with .Values.node.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.node.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + lifecycle: + preStop: + exec: + command: ["/bin/aws-ebs-csi-driver", "pre-stop-hook"] + - name: node-driver-registrar + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + {{- range .Values.sidecars.nodeDriverRegistrar.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ printf "%s/plugins/ebs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: probe-dir + mountPath: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.additionalArgs }} + - {{ . }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: cryptsetup + hostPath: + path: /run/cryptsetup + type: Directory + - name: probe-dir + {{- if .Values.node.probeDirVolume }} + {{- toYaml .Values.node.probeDirVolume | nindent 10 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.node.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml index 816fdf66e..bff6577b3 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-attacher.yaml @@ -21,3 +21,6 @@ rules: - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments/status" ] verbs: [ "patch" ] + {{- with .Values.sidecars.attacher.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml index 3ca368efb..2b7295aaf 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-csi-node.yaml @@ -8,4 +8,10 @@ metadata: rules: - apiGroups: [""] resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] verbs: ["get"] diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml index 0fb7ded0f..b67c65844 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-provisioner.yaml @@ -30,9 +30,12 @@ rules: - apiGroups: [ "" ] resources: [ "nodes" ] verbs: [ "get", "list", "watch" ] - - apiGroups: [ "coordination.k8s.io" ] - resources: [ "leases" ] - verbs: [ "get", "watch", "list", "delete", "update", "create" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments" ] verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattributesclasses" ] + verbs: [ "get" ] + {{- with .Values.sidecars.provisioner.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml index 065f3aba2..81858af34 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-resizer.yaml @@ -29,3 +29,9 @@ rules: - apiGroups: [ "" ] resources: [ "pods" ] verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattributesclasses" ] + verbs: [ "get", "list", "watch" ] + {{- with .Values.sidecars.resizer.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml index 38e688a8a..697e818d9 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/clusterrole-snapshotter.yaml @@ -24,4 +24,7 @@ rules: verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ] - apiGroups: [ "snapshot.storage.k8s.io" ] resources: [ "volumesnapshotcontents/status" ] - verbs: [ "update" ] + verbs: [ "update", "patch" ] + {{- with .Values.sidecars.snapshotter.additionalClusterRoleRules }} + {{- . | toYaml | nindent 2 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml index 0d79331ac..89468b1ca 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/controller.yaml @@ -6,8 +6,15 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.controller.replicaCount }} + {{- if or (kindIs "float64" .Values.controller.revisionHistoryLimit) (kindIs "int64" .Values.controller.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }} + {{- end }} {{- with .Values.controller.updateStrategy }} strategy: {{- toYaml . | nindent 4 }} @@ -26,7 +33,7 @@ spec: {{- end }} {{- if .Values.controller.podAnnotations }} annotations: - {{- toYaml .Values.controller.podAnnotations | nindent 8 }} + {{- tpl ( .Values.controller.podAnnotations | toYaml ) . | nindent 8 }} {{- end }} spec: nodeSelector: @@ -75,7 +82,7 @@ spec: {{- if .Values.controller.extraVolumeTags }} {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} {{- end }} - {{- with .Values.controller.k8sTagClusterId }} + {{- with (tpl (default "" .Values.controller.k8sTagClusterId) . ) }} - --k8s-tag-cluster-id={{ . }} {{- end }} {{- if and (.Values.controller.enableMetrics) (not .Values.controller.httpEndpoint) }} @@ -87,9 +94,18 @@ spec: {{- if .Values.controller.sdkDebugLog }} - --aws-sdk-debug-log=true {{- end}} + {{- if .Values.controller.batching }} + - --batching=true + {{- end}} {{- with .Values.controller.loggingFormat }} - --logging-format={{ . }} {{- end }} + {{- with .Values.controller.userAgentExtra }} + - --user-agent-extra={{ . }} + {{- end }} + {{- if .Values.controller.otelTracing }} + - --enable-otel-tracing=true + {{- end}} - --v={{ .Values.controller.logLevel }} {{- range .Values.controller.additionalArgs }} - {{ . }} @@ -101,18 +117,20 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + {{- with .Values.awsAccessSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: aws-secret - key: key_id + name: {{ .name }} + key: {{ .keyId }} optional: true - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: aws-secret - key: access_key + name: {{ .name }} + key: {{ .accessKey }} optional: true + {{- end }} - name: AWS_EC2_ENDPOINT valueFrom: configMapKeyRef: @@ -129,10 +147,16 @@ spec: {{- with .Values.controller.env }} {{- . | toYaml | nindent 12 }} {{- end }} - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} + {{- with .Values.controller.otelTracing }} + - name: OTEL_SERVICE_NAME + value: {{ .otelServiceName }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otelExporterEndpoint }} {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -176,6 +200,9 @@ spec: image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.provisioner.image.repository .Values.sidecars.provisioner.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.provisioner.image.pullPolicy }} args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.provisioner.additionalArgs)) }} + - --timeout=60s + {{- end }} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.provisioner.logLevel }} - --feature-gates=Topology=true @@ -195,6 +222,14 @@ spec: {{- end }} {{- end }} - --default-fstype={{ .Values.controller.defaultFsType }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.provisioner.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- range .Values.sidecars.provisioner.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -204,10 +239,10 @@ spec: {{- with .Values.sidecars.provisioner.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -223,6 +258,9 @@ spec: image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.attacher.image.repository .Values.sidecars.attacher.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.attacher.image.pullPolicy }} args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.attacher.additionalArgs)) }} + - --timeout=60s + {{- end }} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.attacher.logLevel }} - --leader-election={{ .Values.sidecars.attacher.leaderElection.enabled | required "leader election state for csi-attacher is required, must be set to true || false." }} @@ -237,6 +275,14 @@ spec: - --leader-election-retry-period={{ .Values.sidecars.attacher.leaderElection.retryPeriod }} {{- end }} {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.attacher.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- range .Values.sidecars.attacher.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -246,10 +292,10 @@ spec: {{- with .Values.sidecars.attacher.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -271,6 +317,14 @@ spec: {{- if .Values.controller.extraCreateMetadata }} - --extra-create-metadata {{- end}} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.snapshotter.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- range .Values.sidecars.snapshotter.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -280,10 +334,10 @@ spec: {{- with .Values.sidecars.snapshotter.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -296,13 +350,94 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} + {{- if (.Values.controller.volumeModificationFeature).enabled }} + - name: volumemodifier + image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.volumemodifier.image.repository .Values.sidecars.volumemodifier.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.volumemodifier.image.pullPolicy }} + args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.volumemodifier.additionalArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.volumemodifier.logLevel }} + - --leader-election={{ .Values.sidecars.volumemodifier.leaderElection.enabled | required "leader election state for csi-volumemodifier is required, must be set to true || false." }} + {{- if .Values.sidecars.volumemodifier.leaderElection.enabled }} + {{- if .Values.sidecars.volumemodifier.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.volumemodifier.leaderElection.leaseDuration }} + {{- end }} + {{- if .Values.sidecars.volumemodifier.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.volumemodifier.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.volumemodifier.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.volumemodifier.leaderElection.retryPeriod }} + {{- end }} + {{- end }} + {{- range .Values.sidecars.volumemodifier.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.volumemodifier.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.volumemodifier.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.volumemodifier.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} - name: csi-resizer image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.resizer.image.repository .Values.sidecars.resizer.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.resizer.image.pullPolicy }} args: + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.resizer.additionalArgs)) }} + - --timeout=60s + {{- end }} - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.resizer.logLevel }} - --handle-volume-inuse-error=false + {{- with .Values.sidecars.resizer.leaderElection }} + - --leader-election={{ .enabled | default true }} + {{- if .leaseDuration }} + - --leader-election-lease-duration={{ .leaseDuration }} + {{- end }} + {{- if .renewDeadline }} + - --leader-election-renew-deadline={{ .renewDeadline }} + {{- end }} + {{- if .retryPeriod }} + - --leader-election-retry-period={{ .retryPeriod }} + {{- end }} + {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-workers)" (join " " .Values.sidecars.resizer.additionalArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --workers=100 + {{- end }} + {{- range .Values.sidecars.resizer.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock @@ -312,10 +447,10 @@ spec: {{- with .Values.sidecars.resizer.env }} {{- . | toYaml | nindent 12 }} {{- end }} + {{- with .Values.controller.envFrom }} envFrom: - {{- with .Values.controller.envFrom }} {{- . | toYaml | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -332,10 +467,13 @@ spec: imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} + {{- range .Values.sidecars.livenessProbe.additionalArgs }} + - {{ . }} {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -355,7 +493,15 @@ spec: {{- end }} volumes: - name: socket-dir + {{- if .Values.controller.socketDirVolume }} + {{- toYaml .Values.controller.socketDirVolume | nindent 10 }} + {{- else }} emptyDir: {} + {{- end }} {{- with .Values.controller.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.controller.dnsConfig }} + dnsConfig: + {{- toYaml .Values.controller.dnsConfig | nindent 4 }} + {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml new file mode 100644 index 000000000..a58595726 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/ebs-csi-default-sc.yaml @@ -0,0 +1,11 @@ +{{- if .Values.defaultStorageClass.enabled }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ebs-csi-default-sc + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml index 1dcdf4ddc..d68bd7ab9 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/metrics.yaml @@ -37,6 +37,6 @@ spec: endpoints: - targetPort: 3301 path: /metrics - interval: 15s + interval: {{ .Values.controller.serviceMonitor.interval | default "15s"}} {{- end }} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml index 921b51cfb..9a2c2c81a 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node-windows.yaml @@ -1,184 +1,13 @@ -{{- if .Values.node.enableWindows }} -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node-windows - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - updateStrategy: - {{ toYaml .Values.node.updateStrategy | nindent 4 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podLabels }} - {{- toYaml .Values.node.podLabels | nindent 8 }} - {{- end }} - {{- with .Values.node.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: windows - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- else }} - {{- with .Values.node.tolerations }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - containers: - - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- with .Values.node.volumeAttachLimit }} - - --volume-attach-limit={{ . }} - {{- end }} - {{- with .Values.node.loggingFormat }} - - --logging-format={{ . }} - {{- end }} - - --v={{ .Values.node.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.node.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: C:\var\lib\kubelet - mountPropagation: "None" - - name: plugin-dir - mountPath: C:\csi - - name: csi-proxy-disk-pipe - mountPath: \\.\pipe\csi-proxy-disk-v1 - - name: csi-proxy-volume-pipe - mountPath: \\.\pipe\csi-proxy-volume-v1 - - name: csi-proxy-filesystem-pipe - mountPath: \\.\pipe\csi-proxy-filesystem-v1 - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: node-driver-registrar - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} - env: - - name: ADDRESS - value: unix:/csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: C:\var\lib\kubelet\plugins\aws.csi.confidential.cloud\csi.sock - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - livenessProbe: - exec: - command: - - /csi-node-driver-registrar.exe - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - volumeMounts: - - name: plugin-dir - mountPath: C:\csi - - name: registration-dir - mountPath: C:\registration - - name: probe-dir - mountPath: C:\var\lib\kubelet\plugins\aws.csi.confidential.cloud - {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=unix:/csi/csi.sock - volumeMounts: - - name: plugin-dir - mountPath: C:\csi - {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: C:\var\lib\kubelet - type: Directory - - name: plugin-dir - hostPath: - path: C:\var\lib\kubelet\plugins\aws.csi.confidential.cloud - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: C:\var\lib\kubelet\plugins_registry - type: Directory - - name: csi-proxy-disk-pipe - hostPath: - path: \\.\pipe\csi-proxy-disk-v1 - type: "" - - name: csi-proxy-volume-pipe - hostPath: - path: \\.\pipe\csi-proxy-volume-v1 - type: "" - - name: csi-proxy-filesystem-pipe - hostPath: - path: \\.\pipe\csi-proxy-filesystem-v1 - type: "" - - name: probe-dir - emptyDir: {} +{{$defaultArgs := dict + "NodeName" "ebs-csi-node" +}} +{{- include "node-windows" (deepCopy $ | mustMerge $defaultArgs) -}} +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "ebs-csi-node-%s" $name) + "Values" (dict + "node" (deepCopy $.Values.node | mustMerge $values) + ) +}} +{{- include "node-windows" (deepCopy $ | mustMerge $args) -}} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml index ecc6412d8..a891513b6 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/node.yaml @@ -1,206 +1,46 @@ -# Node Service -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node - namespace: {{ .Release.Namespace }} - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - updateStrategy: - {{- toYaml .Values.node.updateStrategy | nindent 4 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podLabels }} - {{- toYaml .Values.node.podLabels | nindent 8 }} - {{- end }} - {{- with .Values.node.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.node.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.node.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.node.serviceAccount.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- else }} - {{- with .Values.node.tolerations }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - {{- with .Values.node.securityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- with .Values.node.volumeAttachLimit }} - - --volume-attach-limit={{ . }} - {{- end }} - {{- with .Values.node.loggingFormat }} - - --logging-format={{ . }} - {{- end }} - - "--kms-addr={{ .Values.kms.keyServiceName }}.{{ .Values.kms.keyServiceNamespace | default .Release.Namespace }}:{{ .Values.kms.keyServicePort }}" - - --v={{ .Values.node.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.node.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: {{ .Values.node.kubeletPath }} - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: device-dir - mountPath: /dev - - name: cryptsetup - mountPath: /run/cryptsetup - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.node.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.node.containerSecurityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: node-driver-registrar - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: {{ printf "%s/plugins/aws.csi.confidential.cloud/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} - {{- if .Values.proxy.http_proxy }} - {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.env }} - {{- . | toYaml | nindent 12 }} - {{- end }} - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} - {{- end }} - livenessProbe: - exec: - command: - - /csi-node-driver-registrar - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - initialDelaySeconds: 30 - timeoutSeconds: 15 - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: probe-dir - mountPath: {{ printf "%s/plugins/aws.csi.confidential.cloud/" (trimSuffix "/" .Values.node.kubeletPath) }} - {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} - args: - - --csi-address=/csi/csi.sock - envFrom: - {{- with .Values.controller.envFrom }} - {{- . | toYaml | nindent 12 }} - {{- end }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.sidecars.livenessProbe.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: {{ .Values.node.kubeletPath }} - type: Directory - - name: plugin-dir - hostPath: - path: {{ printf "%s/plugins/aws.csi.confidential.cloud/" (trimSuffix "/" .Values.node.kubeletPath) }} - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} - type: Directory - - name: device-dir - hostPath: - path: /dev - type: Directory - - name: probe-dir - emptyDir: {} - - name: cryptsetup - hostPath: - path: /run/cryptsetup - type: Directory +{{$defaultArgs := dict + "NodeName" "ebs-csi-node" +}} +{{- include "node" (deepCopy $ | mustMerge $defaultArgs) -}} +{{- range $name, $values := .Values.additionalDaemonSets }} +{{$args := dict + "NodeName" (printf "ebs-csi-node-%s" $name) + "Values" (dict + "node" (deepCopy $.Values.node | mustMerge $values) + ) +}} +{{- include "node" (deepCopy $ | mustMerge $args) -}} +{{- end }} +{{- if .Values.a1CompatibilityDaemonSet }} +{{$args := dict + "NodeName" "ebs-csi-node-a1compat" + "Values" (dict + "image" (dict + "tag" (printf "%s-a1compat" (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString))) + ) + "node" (dict + "affinity" (dict + "nodeAffinity" (dict + "requiredDuringSchedulingIgnoredDuringExecution" (dict + "nodeSelectorTerms" (list + (dict "matchExpressions" (list + (dict + "key" "eks.amazonaws.com/compute-type" + "operator" "NotIn" + "values" (list "fargate") + ) + (dict + "key" "node.kubernetes.io/instance-type" + "operator" "In" + "values" (list "a1.medium" "a1.large" "a1.xlarge" "a1.2xlarge" "a1.4xlarge") + ) + )) + ) + ) + ) + ) + ) + ) +}} +{{- include "node" (deepCopy $ | mustMerge $args) -}} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml index 6f73fa222..0a1e97cc0 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/poddisruptionbudget-controller.yaml @@ -1,8 +1,4 @@ -{{- if .Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }} apiVersion: policy/v1 -{{- else }} -apiVersion: policy/v1beta1 -{{- end }} kind: PodDisruptionBudget metadata: name: ebs-csi-controller diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml new file mode 100644 index 000000000..1ec62bb49 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/role-leases.yaml @@ -0,0 +1,11 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: {{ .Release.Namespace }} + name: ebs-csi-leases-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml new file mode 100644 index 000000000..88fded8a3 --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/rolebinding-leases.yaml @@ -0,0 +1,15 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-leases-rolebinding + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: ebs-csi-leases-role + apiGroup: rbac.authorization.k8s.io diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml index a5b1102b4..d819f5493 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -15,4 +15,7 @@ metadata: #annotations: # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role {{- end }} +{{- if .Values.controller.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.controller.serviceAccount.automountServiceAccountToken }} +{{- end }} {{- end -}} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml index fb85abedf..9f3c7c7e1 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/serviceaccount-csi-node.yaml @@ -10,4 +10,7 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} +{{- if .Values.node.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.node.serviceAccount.automountServiceAccountToken }} +{{- end }} {{- end -}} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml index 0db3046aa..59551898e 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/templates/volumesnapshotclass.yaml @@ -8,6 +8,9 @@ metadata: {{- with .annotations }} annotations: {{- . | toYaml | trim | nindent 4 }} {{- end }} + {{- with .labels }} + labels: {{- . | toYaml | trim | nindent 4 }} + {{- end }} driver: aws.csi.confidential.cloud deletionPolicy: {{ .deletionPolicy }} {{- with .parameters }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml index defdd4d83..0c4a68857 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/aws-csi-driver/values.yaml @@ -5,7 +5,7 @@ image: repository: ghcr.io/edgelesssys/constellation/aws-csi-driver # Overrides the image tag whose default is v{{ .Chart.AppVersion }} - tag: "v1.1.0@sha256:3e2c394f2397455516948efcc9a4b87cbaeabf14df31702d5905ce08a53995f2" + tag: "v1.2.0@sha256:9477e8ed37989c46963d57d24de5ddbc7ab965b1685d709d06c1ae536b23b5b1" pullPolicy: Always # -- Custom labels to add into metadata @@ -24,8 +24,12 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: "v3.4.1-eks-1-26-7@sha256:adfcb04433d1824f62dde0365877d0f7b7a2eaebc45670cbab7e0c1f07ba0607" + tag: "v4.0.1-eks-1-30-4@sha256:0cf0d02211632c6b947f110e9f3f13f782eea1cfb7b990191d78ad032b2c2d77" logLevel: 2 + # Additional parameters provided by external-provisioner. + additionalArgs: [] + # Grant additional permissions to external-provisioner + additionalClusterRoleRules: resources: {} # Tune leader lease election for csi-provisioner. # Leader election is on by default. @@ -45,7 +49,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher - tag: "v4.2.0-eks-1-26-7@sha256:4b0d6e8758a0213ec942381b9577d2b3e971b545dc9e3fb59973f7992763d85f" + tag: "v4.5.1-eks-1-30-4@sha256:d68034351f65101d2a8506a5c583c5c923238aa93ba9719e779c0eb6a4b33993" # Tune leader lease election for csi-attacher. # Leader election is on by default. leaderElection: @@ -57,6 +61,10 @@ sidecars: # renewDeadline: "10s" # retryPeriod: "5s" logLevel: 2 + # Additional parameters provided by external-attacher. + additionalArgs: [] + # Grant additional permissions to external-attacher + additionalClusterRoleRules: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -68,8 +76,12 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter - tag: "v6.2.1-eks-1-26-7@sha256:b8071f45885f1838387edb04a1d164680dcec8d656de682624ddc59d30ba660b" + tag: "v7.0.2-eks-1-30-4@sha256:9a33488c2cd691d4df454fbc0118e532cbd8aacf99856bdf395507fdae2421dc" logLevel: 2 + # Additional parameters provided by csi-snapshotter. + additionalArgs: [] + # Grant additional permissions to csi-snapshotter + additionalClusterRoleRules: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -78,7 +90,9 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe - tag: "v2.9.0-eks-1-26-7@sha256:d9e11b42ae5f4f2f7ea9034e68040997cdbb04ae9e188aa897f76ae92698d78a" + tag: "v2.12.0-eks-1-30-4@sha256:665d64a8e1124ecd95e08626ddd140154be30a95c6574d423d66cf262d28cc9c" + # Additional parameters provided by livenessprobe. + additionalArgs: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -88,8 +102,22 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer - tag: "v1.7.0-eks-1-26-7@sha256:81672f19d1da5cdff8d2068d8d69776067a1e5c31537ab3282d95dff34d581b6" + tag: "v1.10.1-eks-1-30-4@sha256:2aef6bf851fc3fa8e03c7a3efc9d3adb2ae1cb1746f88fb8a7559f8ca44bf188" + # Tune leader lease election for csi-resizer. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" logLevel: 2 + # Additional parameters provided by external-resizer. + additionalArgs: [] + # Grant additional permissions to external-resizer + additionalClusterRoleRules: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -99,8 +127,40 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: "v2.7.0-eks-1-26-7@sha256:6ad0cae2ae91453f283a44e9b430e475b8a9fa3d606aec9a8b09596fffbcd2c9" + tag: "v2.10.1-eks-1-30-4@sha256:518ed9cba6258735a25d2b896dc65d34a41e22f6785550a7e24e2f2dbd6a48b5" logLevel: 2 + # Additional parameters provided by node-driver-registrar. + additionalArgs: [] + resources: {} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 + volumemodifier: + env: [] + image: + pullPolicy: IfNotPresent + repository: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s + tag: "v0.3.0@sha256:c9e4e64e721b8fea4ba34502ac9f8fb83bd1571117276f553ac4595e0c5a3bf8" + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + logLevel: 2 + # Additional parameters provided by volume-modifier-for-k8s. + additionalArgs: [] resources: {} securityContext: readOnlyRootFilesystem: true @@ -114,7 +174,16 @@ imagePullSecrets: [] nameOverride: fullnameOverride: +awsAccessSecret: + name: aws-secret + keyId: key_id + accessKey: access_key + controller: + batching: true + volumeModificationFeature: + enabled: false + # Additional parameters provided by aws-ebs-csi-driver controller. additionalArgs: [] sdkDebugLog: false loggingFormat: text @@ -165,12 +234,15 @@ controller: # Additional labels for ServiceMonitor object labels: release: prometheus + interval: "15s" # If set to true, AWS API call metrics will be exported to the following # TCP endpoint: "0.0.0.0:3301" # --- # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). k8sTagClusterId: logLevel: 2 + userAgentExtra: "helm" + deploymentAnnotations: {} nodeSelector: node-role.kubernetes.io/control-plane: "" podAnnotations: {} @@ -182,6 +254,9 @@ controller: # region: us-east-1 region: replicaCount: 2 + revisionHistoryLimit: 10 + socketDirVolume: + emptyDir: {} updateStrategy: type: RollingUpdate rollingUpdate: @@ -195,13 +270,15 @@ controller: cpu: 10m memory: 40Mi limits: - cpu: 100m memory: 256Mi serviceAccount: # A service account will be created for you if set to true. Set to false if you want to use your own. create: true name: ebs-csi-controller-sa annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role + automountServiceAccountToken: true tolerations: - key: CriticalAddonsOnly operator: Exists @@ -235,8 +312,18 @@ controller: runAsUser: 1000 runAsGroup: 1000 fsGroup: 1000 + # Add additional volume mounts on the controller with controller.volumes and controller.volumeMounts volumes: [] + # Add additional volumes to be mounted onto the controller: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # --- # securityContext on the controller container (see sidecars for securityContext on sidecar containers) containerSecurityContext: readOnlyRootFilesystem: true @@ -249,6 +336,13 @@ controller: # - name: wait # image: busybox # command: [ 'sh', '-c', "sleep 20" ] + # Enable opentelemetry tracing for the plugin running on the daemonset + otelTracing: {} + # otelServiceName: ebs-csi-controller + # otelExporterEndpoint: "http://localhost:4317" + + # Enable dnsConfig for the controller and node pods + dnsConfig: {} node: env: [] @@ -257,16 +351,26 @@ node: loggingFormat: text logLevel: 2 priorityClassName: + additionalArgs: [] affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: node.kubernetes.io/instance-type + operator: NotIn + values: + - a1.medium + - a1.large + - a1.xlarge + - a1.2xlarge + - a1.4xlarge nodeSelector: {} + daemonSetAnnotations: {} podAnnotations: {} podLabels: {} tolerateAllTaints: true @@ -279,19 +383,32 @@ node: cpu: 10m memory: 40Mi limits: - cpu: 100m memory: 256Mi + revisionHistoryLimit: 10 + probeDirVolume: + emptyDir: {} serviceAccount: create: true name: ebs-csi-node-sa annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role + automountServiceAccountToken: true + # Enable the linux daemonset creation + enableLinux: true enableWindows: false + # The number of attachment slots to reserve for system use (and not to be used for CSI volumes) + # When this parameter is not specified (or set to -1), the EBS CSI Driver will attempt to determine the number of reserved slots via heuristic + # Cannot be specified at the same time as `node.volumeAttachLimit` + reservedVolumeAttachments: # The "maximum number of attachable volumes" per node + # Cannot be specified at the same time as `node.reservedVolumeAttachments` volumeAttachLimit: updateStrategy: type: RollingUpdate rollingUpdate: maxUnavailable: "10%" + hostNetwork: false # securityContext on the node pod securityContext: # The node pod must be run as root to bind to the registration/driver sockets @@ -299,10 +416,38 @@ node: runAsUser: 0 runAsGroup: 0 fsGroup: 0 + # Add additional volume mounts on the node pods with node.volumes and node.volumeMounts + volumes: [] + # Add additional volumes to be mounted onto the node pods: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory + volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # --- # securityContext on the node container (see sidecars for securityContext on sidecar containers) containerSecurityContext: readOnlyRootFilesystem: true privileged: true + # Enable opentelemetry tracing for the plugin running on the daemonset + otelTracing: {} + # otelServiceName: ebs-csi-node + # otelExporterEndpoint: "http://localhost:4317" + +additionalDaemonSets: + # Additional node DaemonSets, using the node config structure + # See docs/additional-daemonsets.md for more information + # + # example: + # nodeSelector: + # node.kubernetes.io/instance-type: c5.large + # volumeAttachLimit: 15 + +# Enable compatibility for the A1 instance family via use of an AL2-based image in a separate DaemonSet +# a1CompatibilityDaemonSet: true # Create Constellation default StorageClasses createStorageClass: true @@ -323,12 +468,18 @@ storageClasses: [] # parameters: # encrypted: "true" +defaultStorageClass: + enabled: false + volumeSnapshotClasses: [] # Add VolumeSnapshotClass resources like: # - name: ebs-vsc # # annotation metadata # annotations: # snapshot.storage.kubernetes.io/is-default-class: "true" +# # label metadata +# labels: +# my-label-is: supercool # # deletionPolicy must be specified # deletionPolicy: Delete # parameters: @@ -337,3 +488,8 @@ volumeSnapshotClasses: [] # Intended for use with older clusters that cannot easily replace the CSIDriver object # This parameter should always be false for new installations useOldCSIDriver: false + +helmTester: + enabled: true + # Supply a custom image to the ebs-csi-driver-test pod in helm-tester.yaml + image: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240311-b09cdeb92c-master" diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/Chart.yaml index fbab5e66d..3ce5248a0 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: "v1.3.0" +appVersion: "v1.4.0" description: Azure disk Container Storage Interface (CSI) Storage Plugin with on-node encryption support name: azuredisk-csi-driver -version: v1.3.0 +version: v1.4.0 diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml index f7ac6f2db..645a4da75 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml @@ -120,6 +120,7 @@ spec: - "-leader-election" - "--leader-election-namespace={{ .Release.Namespace }}" - "-v=2" + - "--timeout=1200s" env: - name: ADDRESS value: /csi/csi.sock @@ -157,7 +158,11 @@ spec: args: - --csi-address=/csi/csi.sock - --probe-timeout=3s +{{- if eq .Values.controller.hostNetwork true }} + - --http-endpoint=localhost:{{ .Values.controller.livenessProbe.healthPort }} +{{- else }} - --health-port={{ .Values.controller.livenessProbe.healthPort }} +{{- end }} - --v=2 volumeMounts: - name: socket-dir @@ -197,18 +202,29 @@ spec: - "--enable-traffic-manager={{ .Values.controller.enableTrafficManager }}" - "--traffic-manager-port={{ .Values.controller.trafficManagerPort }}" - "--enable-otel-tracing={{ .Values.controller.otelTracing.enabled }}" + - "--check-disk-lun-collision=true" + {{- range $value := .Values.controller.extraArgs }} + - {{ $value | quote }} + {{- end }} ports: - - containerPort: {{ .Values.controller.livenessProbe.healthPort }} - name: healthz - protocol: TCP - containerPort: {{ .Values.controller.metricsPort }} name: metrics protocol: TCP +{{- if ne .Values.controller.hostNetwork true }} + - containerPort: {{ .Values.controller.livenessProbe.healthPort }} + name: healthz + protocol: TCP +{{- end }} livenessProbe: failureThreshold: 5 httpGet: path: /healthz +{{- if eq .Values.controller.hostNetwork true }} + host: localhost + port: {{ .Values.controller.livenessProbe.healthPort }} +{{- else }} port: healthz +{{- end }} initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml index 50d3b795c..9d9c368d9 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml @@ -74,7 +74,11 @@ spec: args: - --csi-address=/csi/csi.sock - --probe-timeout=3s +{{- if eq .Values.linux.hostNetwork true }} + - --http-endpoint=localhost:{{ .Values.node.livenessProbe.healthPort }} +{{- else }} - --health-port={{ .Values.node.livenessProbe.healthPort }} +{{- end }} - --v=2 resources: {{- toYaml .Values.linux.resources.livenessProbe | nindent 12 }} - name: node-driver-registrar @@ -131,15 +135,22 @@ spec: - "--get-nodeid-from-imds={{ .Values.node.getNodeIDFromIMDS }}" - "--enable-otel-tracing={{ .Values.linux.otelTracing.enabled }}" - "--kms-addr={{ .Values.global.keyServiceName }}.{{ .Values.global.keyServiceNamespace | default .Release.Namespace }}:{{ .Values.global.keyServicePort }}" +{{- if ne .Values.linux.hostNetwork true }} ports: - containerPort: {{ .Values.node.livenessProbe.healthPort }} name: healthz protocol: TCP +{{- end }} livenessProbe: failureThreshold: 5 httpGet: path: /healthz +{{- if eq .Values.linux.hostNetwork true }} + host: localhost + port: {{ .Values.node.livenessProbe.healthPort }} +{{- else }} port: healthz +{{- end }} initialDelaySeconds: 30 timeoutSeconds: 10 periodSeconds: 30 diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/values.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/values.yaml index 944663770..18faf65c3 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/values.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/azuredisk-csi-driver/values.yaml @@ -2,27 +2,27 @@ image: baseRepo: mcr.microsoft.com azuredisk: repository: ghcr.io/edgelesssys/constellation/azure-csi-driver - tag: v1.3.0@sha256:1e798f066ef78c293c4c87a31677f8948be4c8709980135969b73a9d7a46ca71 + tag: v1.4.0@sha256:e41b09d2735cb7410e2bf7abe9ca2166aa5a949d6c6e2ac570773b5d041797f1 pullPolicy: IfNotPresent csiProvisioner: repository: /oss/kubernetes-csi/csi-provisioner - tag: v3.5.0@sha256:fdf70099aa1538d1c2164976cf6d158ef8b3a5ee63db10bf0085de4ec66f59b4 + tag: v4.0.0@sha256:beadfb2cfa02f8bbb2efd88261a673023527cf51ebe7894daef82c4d928264a5 pullPolicy: IfNotPresent csiAttacher: repository: /oss/kubernetes-csi/csi-attacher - tag: v4.3.0@sha256:4306b80bfe8caea3fe53f6d1c15807c745be3072553ff508fc4f61da8f4a0c10 + tag: v4.5.0@sha256:172a9140780701b2223b7296729fc6cc3be8c86d0cfd2d0452e495f5ea28f51f pullPolicy: IfNotPresent csiResizer: repository: /oss/kubernetes-csi/csi-resizer - tag: v1.8.0@sha256:6f0e8c9f3d0bdcf7a5fb5e404276ffac624033099d7687c8080692bcb6d13cd1 + tag: v1.9.3@sha256:e20dc798f529436d2c861dd66bc7fcfa17623b562a2a65474aab38fb77c9824a pullPolicy: IfNotPresent livenessProbe: repository: /oss/kubernetes-csi/livenessprobe - tag: v2.10.0@sha256:3aeac313cffdb7db80b733539427f2533a3f662bf538e7b6434b0f898ceb701b + tag: v2.12.0@sha256:c762188c45d1b9bc9144b694b85313d5e49c741935a81d5b94fd7db978a40ae1 pullPolicy: IfNotPresent nodeDriverRegistrar: repository: /oss/kubernetes-csi/csi-node-driver-registrar - tag: v2.8.0@sha256:af6bf1b5ff310d4dc02cf8276be9b06014318f7ee31238b5fa278febd1a10ca9 + tag: v2.10.0@sha256:136e3a4a5897f111d1dedd404a5717ee7ff2f215e5fe878abdf4ce00c2292280 pullPolicy: IfNotPresent serviceAccount: @@ -140,11 +140,11 @@ snapshot: image: csiSnapshotter: repository: /oss/kubernetes-csi/csi-snapshotter - tag: v6.2.2 + tag: v6.3.3 pullPolicy: IfNotPresent csiSnapshotController: repository: /oss/kubernetes-csi/snapshot-controller - tag: v6.2.2 + tag: v6.3.3 pullPolicy: IfNotPresent snapshotController: name: csi-snapshot-controller diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/Chart.yaml index ed8008238..0380cc531 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -version: 1.3.0 -appVersion: "v1.3.0" +version: 1.4.0 +appVersion: "v1.4.0" description: GCP Compute Persistent Disk Container Storage Interface (CSI) Storage Plugin with on-node encryption support name: gcp-compute-persistent-disk-csi-driver diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/values.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/values.yaml index e620dde5a..2ac7e6b5e 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/values.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/gcp-compute-persistent-disk-csi-driver/values.yaml @@ -1,28 +1,28 @@ image: csiProvisioner: repo: registry.k8s.io/sig-storage/csi-provisioner - tag: v3.4.0@sha256:e468dddcd275163a042ab297b2d8c2aca50d5e148d2d22f3b6ba119e2f31fa79 + tag: v3.6.3@sha256:10624570c0aceb03f55f1eb07147b0c537e4676869cca2e9bd4bab113f810ac4 pullPolicy: IfNotPresent csiAttacher: repo: registry.k8s.io/sig-storage/csi-attacher - tag: v4.2.0@sha256:34cf9b32736c6624fc9787fb149ea6e0fbeb45415707ac2f6440ac960f1116e6 + tag: v4.4.3@sha256:d7325367ab72b2d469a5091d87b4fc01142d2d13d1a28b2defbbe3e6fdbc4611 pullPolicy: IfNotPresent csiResizer: repo: registry.k8s.io/sig-storage/csi-resizer - tag: v1.7.0@sha256:3a7bdf5d105783d05d0962fa06ca53032b01694556e633f27366201c2881e01d + tag: v1.9.3@sha256:3c116f543f0590aeff3299c8bb0683f250817d11a77d9e9071b15a0bffdabcd9 pullPolicy: IfNotPresent csiSnapshotter: repo: registry.k8s.io/sig-storage/csi-snapshotter - tag: v6.1.0@sha256:291334908ddf71a4661fd7f6d9d97274de8a5378a2b6fdfeb2ce73414a34f82f + tag: v6.3.3@sha256:f1bd6ee18c4021c1c94f29edfab89b49b6a4d1b800936c19dbef2d75f8202f2d pullPolicy: IfNotPresent csiNodeRegistrar: repo: registry.k8s.io/sig-storage/csi-node-driver-registrar - tag: v2.7.0@sha256:4a4cae5118c4404e35d66059346b7fa0835d7e6319ff45ed73f4bba335cf5183 + tag: v2.9.3@sha256:0f64602ea791246712b51df334bbd701a0f31df9950a4cb9c28c059f367baa9e pullPolicy: IfNotPresent gcepdDriver: repo: ghcr.io/edgelesssys/constellation/gcp-csi-driver # CSI driver version is independent of Constellation releases - tag: v1.3.0@sha256:0ecb68f348ed6c287075db00f9c5ea731e7e2db9f2f7511b65391fb6856fe11a + tag: v1.4.0@sha256:53d608aa03dd07059bc04e1f8c64e2feb6fceff50fb0cbe276d31a8652a19bac pullPolicy: IfNotPresent csiController: diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/Chart.yaml index d263e85d6..779770a30 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: v1.0.0 +appVersion: v1.0.2 description: Cinder CSI Chart for OpenStack with on-node encryption support name: openstack-cinder-csi -version: 1.0.0 +version: 1.0.2 diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-deployment.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-deployment.yaml index 9e13f8513..dd9a16882 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-deployment.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-deployment.yaml @@ -5,6 +5,10 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "cinder-csi.controllerplugin.labels" . | nindent 4 }} + annotations: + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.csi.plugin.controllerPlugin.replicas }} strategy: @@ -21,10 +25,18 @@ spec: metadata: labels: {{- include "cinder-csi.controllerplugin.labels" . | nindent 8 }} + annotations: + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccount: csi-cinder-controller-sa + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.podSecurityContext | nindent 8 }} containers: - name: csi-attacher + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.attacher.image.repository }}:{{ .Values.csi.attacher.image.tag }}" imagePullPolicy: {{ .Values.csi.attacher.image.pullPolicy }} args: @@ -46,6 +58,8 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ resources: {{ toYaml .Values.csi.attacher.resources | nindent 12 }} - name: csi-provisioner + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.provisioner.image.repository }}:{{ .Values.csi.provisioner.image.tag }}" imagePullPolicy: {{ .Values.csi.provisioner.image.pullPolicy }} args: @@ -69,6 +83,8 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ resources: {{ toYaml .Values.csi.provisioner.resources | nindent 12 }} - name: csi-snapshotter + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.snapshotter.image.repository }}:{{ .Values.csi.snapshotter.image.tag }}" imagePullPolicy: {{ .Values.csi.snapshotter.image.pullPolicy }} args: @@ -89,6 +105,8 @@ spec: name: socket-dir resources: {{ toYaml .Values.csi.snapshotter.resources | nindent 12 }} - name: csi-resizer + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.resizer.image.repository }}:{{ .Values.csi.resizer.image.tag }}" imagePullPolicy: {{ .Values.csi.resizer.image.pullPolicy }} args: @@ -110,6 +128,8 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ resources: {{ toYaml .Values.csi.resizer.resources | nindent 12 }} - name: liveness-probe + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.livenessprobe.image.repository }}:{{ .Values.csi.livenessprobe.image.tag }}" imagePullPolicy: {{ .Values.csi.livenessprobe.image.pullPolicy }} args: @@ -128,6 +148,8 @@ spec: name: socket-dir resources: {{ toYaml .Values.csi.livenessprobe.resources | nindent 12 }} - name: cinder-csi-plugin + securityContext: + {{- toYaml .Values.csi.plugin.controllerPlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.plugin.image.repository }}:{{ .Values.csi.plugin.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.csi.plugin.image.pullPolicy }} args: @@ -137,6 +159,9 @@ spec: - "--cloud-config=$(CLOUD_CONFIG)" - "--cluster=$(CLUSTER_NAME)" - "--kms-addr={{ .Values.csi.kms.keyServiceName }}.{{ .Values.csi.kms.keyServiceNamespace | default .Release.Namespace }}:{{ .Values.csi.kms.keyServicePort }}" + {{- if .Values.csi.plugin.httpEndpoint.enabled }} + - "--http-endpoint=:{{ .Values.csi.plugin.httpEndpoint.port }}" + {{- end }} {{- if .Values.csi.plugin.extraArgs }} {{- with .Values.csi.plugin.extraArgs }} {{- tpl . $ | trim | nindent 12 }} @@ -153,6 +178,11 @@ spec: - containerPort: 9808 name: healthz protocol: TCP + {{- if .Values.csi.plugin.httpEndpoint.enabled }} + - containerPort: {{ .Values.csi.plugin.httpEndpoint.port }} + name: http + protocol: TCP + {{- end }} # The probe livenessProbe: failureThreshold: {{ .Values.csi.livenessprobe.failureThreshold }} @@ -165,26 +195,33 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi - - name: cloud-config - mountPath: /etc/kubernetes/{{ .Values.secret.filename }} - readOnly: true - subPath: {{ .Values.secret.filename }} + {{- with .Values.csi.plugin.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{ toYaml .Values.csi.plugin.resources | nindent 12 }} volumes: - name: socket-dir emptyDir: - - name: cloud-config {{- if .Values.secret.enabled }} + - name: cloud-config secret: secretName: {{ .Values.secret.name }} - {{- else }} + {{- else if .Values.secret.hostMount }} + - name: cloud-config hostPath: path: /etc/kubernetes {{- end }} + {{- with .Values.csi.plugin.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} affinity: {{ toYaml .Values.csi.plugin.controllerPlugin.affinity | nindent 8 }} nodeSelector: {{ toYaml .Values.csi.plugin.controllerPlugin.nodeSelector | nindent 8 }} tolerations: {{ toYaml .Values.csi.plugin.controllerPlugin.tolerations | nindent 8 }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- with .Values.csi.plugin.controllerPlugin.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-podmonitor.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-podmonitor.yaml new file mode 100644 index 000000000..a1b4ceb4b --- /dev/null +++ b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/controllerplugin-podmonitor.yaml @@ -0,0 +1,22 @@ +{{- if .Values.csi.plugin.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + {{- include "cinder-csi.controllerplugin.labels" . | nindent 4 }} + name: {{ include "cinder-csi.name" . }}-controllerplugin + namespace: {{ .Release.Namespace }} + annotations: + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + podMetricsEndpoints: + - interval: 30s + port: http + scheme: http + jobLabel: component + selector: + matchLabels: + {{- include "cinder-csi.controllerplugin.matchLabels" . | nindent 6 }} +{{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-daemonset.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-daemonset.yaml index dd9f513ac..482f1800e 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-daemonset.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/nodeplugin-daemonset.yaml @@ -5,6 +5,10 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "cinder-csi.nodeplugin.labels" . | nindent 4 }} + annotations: + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: selector: matchLabels: @@ -13,12 +17,18 @@ spec: metadata: labels: {{- include "cinder-csi.nodeplugin.labels" . | nindent 8 }} + annotations: + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccount: csi-cinder-node-sa hostNetwork: true dnsPolicy: ClusterFirstWithHostNet containers: - name: node-driver-registrar + securityContext: + {{- toYaml .Values.csi.plugin.nodePlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.nodeDriverRegistrar.image.repository }}:{{ .Values.csi.nodeDriverRegistrar.image.tag }}" imagePullPolicy: {{ .Values.csi.nodeDriverRegistrar.image.pullPolicy }} args: @@ -46,6 +56,8 @@ spec: mountPath: /registration resources: {{ toYaml .Values.csi.nodeDriverRegistrar.resources | nindent 12 }} - name: liveness-probe + securityContext: + {{- toYaml .Values.csi.plugin.nodePlugin.securityContext | nindent 12 }} image: "{{ .Values.csi.livenessprobe.image.repository }}:{{ .Values.csi.livenessprobe.image.tag }}" imagePullPolicy: {{ .Values.csi.livenessprobe.image.pullPolicy }} args: @@ -106,10 +118,14 @@ spec: - name: pods-probe-dir mountPath: /dev mountPropagation: "HostToContainer" - - name: cloud-config - mountPath: /etc/kubernetes/{{ .Values.secret.filename }} - readOnly: true - subPath: {{ .Values.secret.filename }} + # Edgeless specific mounts for cryptsetup + - name: sys + mountPath: /sys + - name: cryptsetup + mountPath: /run/cryptsetup + {{- with .Values.csi.plugin.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{ toYaml .Values.csi.plugin.resources | nindent 12 }} volumes: - name: socket-dir @@ -124,6 +140,14 @@ spec: hostPath: path: {{ .Values.csi.plugin.nodePlugin.kubeletDir }} type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + - name: cryptsetup + hostPath: + path: /run/cryptsetup + type: Directory # - name: pods-cloud-data # hostPath: # path: /var/lib/cloud/data @@ -132,18 +156,26 @@ spec: hostPath: path: /dev type: Directory - - name: cloud-config {{- if .Values.secret.enabled }} + - name: cloud-config secret: secretName: {{ .Values.secret.name }} - {{- else }} + {{- else if .Values.secret.hostMount }} + - name: cloud-config hostPath: path: /etc/kubernetes {{- end }} + {{- with .Values.csi.plugin.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} affinity: {{ toYaml .Values.csi.plugin.nodePlugin.affinity | nindent 8 }} nodeSelector: {{ toYaml .Values.csi.plugin.nodePlugin.nodeSelector | nindent 8 }} tolerations: {{ toYaml .Values.csi.plugin.nodePlugin.tolerations | nindent 8 }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- with .Values.csi.plugin.nodePlugin.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/secret.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/secret.yaml index b11ef8567..597880c0d 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/secret.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if .Values.secret.create }} +{{- if and (.Values.secret.create) (.Values.secret.enabled) }} apiVersion: v1 kind: Secret metadata: diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/values.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/values.yaml index 40d986102..e6d124384 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/values.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/openstack-cinder-csi/values.yaml @@ -8,7 +8,7 @@ csi: attacher: image: repository: registry.k8s.io/sig-storage/csi-attacher - tag: v4.2.0 + tag: v4.4.2@sha256:11b955fe4da278aa0e8ca9d6fd70758f2aec4b0c1e23168c665ca345260f1882 pullPolicy: IfNotPresent resources: {} extraArgs: {} @@ -16,28 +16,28 @@ csi: topology: "true" image: repository: registry.k8s.io/sig-storage/csi-provisioner - tag: v3.4.1 + tag: v3.6.2@sha256:49b94f975603d85a1820b72b1188e5b351d122011b3e5351f98c49d72719aa78 pullPolicy: IfNotPresent resources: {} extraArgs: {} snapshotter: image: repository: registry.k8s.io/sig-storage/csi-snapshotter - tag: v6.2.1 + tag: v6.3.2@sha256:4c5a1b57e685b2631909b958487f65af7746361346fcd82a8635bea3ef14509d pullPolicy: IfNotPresent resources: {} extraArgs: {} resizer: image: repository: registry.k8s.io/sig-storage/csi-resizer - tag: v1.7.0 + tag: v1.9.2@sha256:e998f22243869416f9860fc6a1fb07d4202eac8846defc1b85ebd015c1207605 pullPolicy: IfNotPresent resources: {} extraArgs: {} livenessprobe: image: repository: registry.k8s.io/sig-storage/livenessprobe - tag: v2.9.0 + tag: v2.11.0@sha256:82adbebdf5d5a1f40f246aef8ddbee7f89dea190652aefe83336008e69f9a89f pullPolicy: IfNotPresent failureThreshold: 5 initialDelaySeconds: 10 @@ -48,7 +48,7 @@ csi: nodeDriverRegistrar: image: repository: registry.k8s.io/sig-storage/csi-node-driver-registrar - tag: v2.6.2 + tag: v2.9.2@sha256:a18e989a93722e43885120e90bc1d0da0740fcbf44bc10403572b368b9800606 pullPolicy: IfNotPresent resources: {} extraArgs: {} @@ -56,17 +56,31 @@ csi: image: repository: ghcr.io/edgelesssys/constellation/cinder-csi-plugin pullPolicy: IfNotPresent - tag: # defaults to .Chart.AppVersion + # CSI driver version is independent of Constellation releases + tag: v1.0.2@sha256:d0b9872378ef5cad9ca5442651df85cf6a4a3a50044018c3541b53f3b7b7480e volumeMounts: - name: cloud-config mountPath: /etc/kubernetes readOnly: true nodePlugin: + dnsPolicy: ClusterFirstWithHostNet + podSecurityContext: {} + securityContext: {} + # capabilities: + # drop: + # - ALL + # seccompProfile: + # type: RuntimeDefault affinity: {} nodeSelector: {} tolerations: - operator: Exists kubeletDir: /var/lib/kubelet + # Allow for specifying internal IP addresses for multiple hostnames + # hostAliases: + # - ip: "10.0.0.1" + # hostnames: + # - "keystone.hostname.com" controllerPlugin: replicas: 1 strategy: @@ -80,10 +94,36 @@ csi: # maxSurge is the maximum number of pods that can be # created over the desired number of pods. maxSurge: 1 + podSecurityContext: {} + # runAsNonRoot: true + # runAsUser: 65532 + # runAsGroup: 65532 + # fsGroup: 65532 + # fsGroupChangePolicy: OnRootMismatch + securityContext: {} + # capabilities: + # drop: + # - ALL + # seccompProfile: + # type: RuntimeDefault + # readOnlyRootFilesystem: true affinity: {} nodeSelector: {} tolerations: [] + # Allow for specifying internal IP addresses for multiple hostnames + # hostAliases: + # - ip: "10.0.0.1" + # hostnames: + # - "keystone.hostname.com" resources: {} + # Enable built-in http server through the http-endpoint flag + httpEndpoint: + enabled: false + port: 8080 + # Create Prometheus Operator PodMonitor. Requires http server above. + # See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor + podMonitor: + enabled: false extraArgs: {} kms: keyServiceName: "key-service" @@ -95,6 +135,12 @@ csi: # for description of individual verbosity levels. logVerbosityLevel: 2 +# the secret should contain the openstack credentials +# there are several options to inject the credentials: +# 1) from kubernetes secret that doesn't exist: set "enabled" and "create" to true, this will create a secret from the values written to "data" down below +# 2) from kubernetes secret that already exists: set "enabled" to true and "create" to false +# 3) from host system path /etc/cloud/cloud.conf: set "enabled" to false and "hostMount" to true +# 4) via agent-injector (e.g. hashicorp vault): set "enabled" and "hostMount" to false, you have to provide credentials on your own by injecting credentials into the pod secret: enabled: true create: false @@ -118,3 +164,6 @@ priorityClassName: "" imagePullSecrets: [] # - name: my-imagepull-secret + +# add annotations to all pods +commonAnnotations: {} diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/Chart.yaml index a6aacc946..be3dde48a 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/Chart.yaml @@ -1,6 +1,9 @@ apiVersion: v2 name: snapshot-controller -description: A chart to deploy the CSI snapshot controller and webhook +description: | + A chart to deploy the CSI snapshot controller and webhook + Snapshot controller source: https://github.com/kubernetes-csi/external-snapshotter/tree/v8.0.1/deploy/kubernetes/snapshot-controller + Snapshot validating webhook source: https://github.com/kubernetes-csi/external-snapshotter/tree/v8.0.1/deploy/kubernetes/webhook-example type: application -version: 6.2.2 -appVersion: "6.2.2" +version: 8.0.1 +appVersion: "8.0.1" diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/admission-configuration.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/admission-configuration.yaml index 95e26f473..ab6159704 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/admission-configuration.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/admission-configuration.yaml @@ -1,3 +1,6 @@ +# Snapshot validating webhook configuration +# Adapted from https://github.com/kubernetes-csi/external-snapshotter/tree/v8.0.1/deploy/kubernetes/webhook-example +# to use cert-manager for serving certificates apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: @@ -10,7 +13,7 @@ webhooks: - apiGroups: ["snapshot.storage.k8s.io"] apiVersions: ["v1"] operations: ["CREATE", "UPDATE"] - resources: ["volumesnapshots", "volumesnapshotcontents", "volumesnapshotclasses"] + resources: ["volumesnapshotclasses"] scope: "*" clientConfig: service: diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/serving-cert.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/serving-cert.yaml index 93c24cec6..00a1935e4 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/serving-cert.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/serving-cert.yaml @@ -6,7 +6,7 @@ metadata: spec: dnsNames: - 'snapshot-validation-service.{{ .Release.Namespace }}.svc' - - 'snapshot-validation-service.{{ .Release.Namespace }}.svc.{{ .Values.kubernetesClusterDomain }}' + - 'snapshot-validation-service.{{ .Release.Namespace }}.svc.cluster.local' issuerRef: kind: Issuer name: snapshot-validation-selfsigned-issuer diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-controller.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-controller.yaml index ae8dbcc4e..bd244b0db 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-controller.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-controller.yaml @@ -16,10 +16,11 @@ spec: selector: matchLabels: app: snapshot-controller - # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable - # in #504 the snapshot-controller will exit after around 7.5 seconds if it - # can't find the v1 CRDs so this value should be greater than that - minReadySeconds: 15 + # The snapshot controller won't be marked as ready if the v1 CRDs are unavailable. + # The flag --retry-crd-interval-max is used to determine how long the controller + # will wait for the CRDs to become available before exiting. The default is 30 seconds + # so minReadySeconds should be set slightly higher than the flag value. + minReadySeconds: 35 strategy: rollingUpdate: maxSurge: 0 diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-webhook.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-webhook.yaml index 861f284d7..8c93b51fc 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-webhook.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/templates/snapshot-webhook.yaml @@ -1,3 +1,6 @@ +# Snapshot validating webhook configuration +# Adapted from https://github.com/kubernetes-csi/external-snapshotter/tree/v8.0.1/deploy/kubernetes/webhook-example +# to use cert-manager for serving certificates --- apiVersion: apps/v1 kind: Deployment diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/values.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/values.yaml index 9c2f219b5..88e453caf 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/values.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-controller/values.yaml @@ -1,15 +1,14 @@ -kubernetesClusterDomain: cluster.local snapshotController: replicas: 2 snapshotController: image: repository: registry.k8s.io/sig-storage/snapshot-controller - tag: v6.2.2@sha256:fb95b65bb88f319f0f7d5397c401a654164f11a191f466b4026fa36085c7141b + tag: v8.2.1@sha256:472fa35a89dadb5a715454fad576ec11aa6f2e8378fc09ae26473d139b77c437 imagePullPolicy: IfNotPresent snapshotWebhook: replicas: 1 webhook: image: repository: registry.k8s.io/sig-storage/snapshot-validation-webhook - tag: v6.2.2@sha256:b5be1e04b7c43352f83e135bd772de05437f8f3a20cb9437875d1a0d4f127440 + tag: v8.1.1@sha256:979842f9a6c23ae1b2ddd26603c27412dfc4d3c027d9cda1cb87a67b91ae9ac8 imagePullPolicy: IfNotPresent diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/Chart.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/Chart.yaml index fd0fc7ae2..aa7a27fe1 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/Chart.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v2 name: snapshot-crds -description: A chart to deploy CSI snapshot CRDs +description: "A chart to deploy CSI snapshot CRDs. Source: https://github.com/kubernetes-csi/external-snapshotter/tree/v8.0.1/client/config/crd" + type: application -version: 6.2.2 -appVersion: "6.2.2" +version: 8.0.1 +appVersion: "8.0.1" diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotclasses.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotclasses.yaml index 56a8e1487..8164952a4 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotclasses.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotclasses.yaml @@ -3,9 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.15.0 name: volumesnapshotclasses.snapshot.storage.k8s.io spec: group: snapshot.storage.k8s.io @@ -34,42 +33,52 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage - system uses when creating a volume snapshot. A specific VolumeSnapshotClass - is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses - are non-namespaced + description: |- + VolumeSnapshotClass specifies parameters that a underlying storage system uses when + creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its + name in a VolumeSnapshot object. + VolumeSnapshotClasses are non-namespaced properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent - created through the VolumeSnapshotClass should be deleted when its bound - VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". - "Retain" means that the VolumeSnapshotContent and its physical snapshot - on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are deleted. + description: |- + deletionPolicy determines whether a VolumeSnapshotContent created through + the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the storage driver that handles this - VolumeSnapshotClass. Required. + description: |- + driver is the name of the storage driver that handles this VolumeSnapshotClass. + Required. type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string + metadata: + type: object parameters: additionalProperties: type: string - description: parameters is a key-value map with storage driver specific - parameters for creating snapshots. These values are opaque to Kubernetes. + description: |- + parameters is a key-value map with storage driver specific parameters for creating snapshots. + These values are opaque to Kubernetes. type: object required: - deletionPolicy diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotcontents.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotcontents.yaml index d6181ed93..cd0c879fc 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotcontents.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshotcontents.yaml @@ -3,9 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.15.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/955" name: volumesnapshotcontents.snapshot.storage.k8s.io spec: group: snapshot.storage.k8s.io @@ -48,7 +47,8 @@ spec: jsonPath: .spec.volumeSnapshotRef.name name: VolumeSnapshot type: string - - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. jsonPath: .spec.volumeSnapshotRef.namespace name: VolumeSnapshotNamespace type: string @@ -58,152 +58,206 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot - object in the underlying storage system + description: |- + VolumeSnapshotContent represents the actual "on-disk" snapshot object in the + underlying storage system properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string + metadata: + type: object spec: - description: spec defines properties of a VolumeSnapshotContent created - by the underlying storage system. Required. + description: |- + spec defines properties of a VolumeSnapshotContent created by the underlying storage system. + Required. properties: deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent - and its physical snapshot on the underlying storage system should - be deleted when its bound VolumeSnapshot is deleted. Supported values - are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are kept. - "Delete" means that the VolumeSnapshotContent and its physical snapshot - on underlying storage system are deleted. For dynamically provisioned - snapshots, this field will automatically be filled in by the CSI - snapshotter sidecar with the "DeletionPolicy" field defined in the - corresponding VolumeSnapshotClass. For pre-existing snapshots, users - MUST specify this field when creating the VolumeSnapshotContent - object. Required. + description: |- + deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on + the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. + For dynamically provisioned snapshots, this field will automatically be filled in by the + CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding + VolumeSnapshotClass. + For pre-existing snapshots, users MUST specify this field when creating the + VolumeSnapshotContent object. + Required. enum: - Delete - Retain type: string driver: - description: driver is the name of the CSI driver used to create the - physical snapshot on the underlying storage system. This MUST be - the same as the name returned by the CSI GetPluginName() call for - that driver. Required. + description: |- + driver is the name of the CSI driver used to create the physical snapshot on + the underlying storage system. + This MUST be the same as the name returned by the CSI GetPluginName() call for + that driver. + Required. type: string source: - description: source specifies whether the snapshot is (or should be) - dynamically provisioned or already exists, and just requires a Kubernetes - object representation. This field is immutable after creation. Required. + description: |- + source specifies whether the snapshot is (or should be) dynamically provisioned + or already exists, and just requires a Kubernetes object representation. + This field is immutable after creation. + Required. properties: snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of - a pre-existing snapshot on the underlying storage system for - which a Kubernetes object representation was (or should be) - created. This field is immutable. - type: string - volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the - volume from which a snapshot should be dynamically taken from. + description: |- + snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on + the underlying storage system for which a Kubernetes object representation + was (or should be) created. This field is immutable. type: string + x-kubernetes-validations: + - message: snapshotHandle is immutable + rule: self == oldSelf + volumeHandle: + description: |- + volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot + should be dynamically taken from. + This field is immutable. + type: string + x-kubernetes-validations: + - message: volumeHandle is immutable + rule: self == oldSelf type: object - oneOf: - - required: ["snapshotHandle"] - - required: ["volumeHandle"] + x-kubernetes-validations: + - message: volumeHandle is required once set + rule: '!has(oldSelf.volumeHandle) || has(self.volumeHandle)' + - message: snapshotHandle is required once set + rule: '!has(oldSelf.snapshotHandle) || has(self.snapshotHandle)' + - message: exactly one of volumeHandle and snapshotHandle must be + set + rule: (has(self.volumeHandle) && !has(self.snapshotHandle)) || (!has(self.volumeHandle) + && has(self.snapshotHandle)) sourceVolumeMode: - description: SourceVolumeMode is the mode of the volume whose snapshot - is taken. Can be either “Filesystem” or “Block”. If not specified, - it indicates the source volume's mode is unknown. This field is - immutable. This field is an alpha field. + description: |- + SourceVolumeMode is the mode of the volume whose snapshot is taken. + Can be either “Filesystem” or “Block”. + If not specified, it indicates the source volume's mode is unknown. + This field is immutable. + This field is an alpha field. type: string + x-kubernetes-validations: + - message: sourceVolumeMode is immutable + rule: self == oldSelf volumeSnapshotClassName: - description: name of the VolumeSnapshotClass from which this snapshot - was (or will be) created. Note that after provisioning, the VolumeSnapshotClass - may be deleted or recreated with different set of values, and as - such, should not be referenced post-snapshot creation. + description: |- + name of the VolumeSnapshotClass from which this snapshot was (or will be) + created. + Note that after provisioning, the VolumeSnapshotClass may be deleted or + recreated with different set of values, and as such, should not be referenced + post-snapshot creation. type: string volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object - to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName - field must reference to this VolumeSnapshotContent's name for the - bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent - object, name and namespace of the VolumeSnapshot object MUST be - provided for binding to happen. This field is immutable after creation. + description: |- + volumeSnapshotRef specifies the VolumeSnapshot object to which this + VolumeSnapshotContent object is bound. + VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to + this VolumeSnapshotContent's name for the bidirectional binding to be valid. + For a pre-existing VolumeSnapshotContent object, name and namespace of the + VolumeSnapshot object MUST be provided for binding to happen. + This field is immutable after creation. Required. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: both spec.volumeSnapshotRef.name and spec.volumeSnapshotRef.namespace + must be set + rule: has(self.name) && has(self.__namespace__) required: - deletionPolicy - driver - source - volumeSnapshotRef type: object + x-kubernetes-validations: + - message: sourceVolumeMode is required once set + rule: '!has(oldSelf.sourceVolumeMode) || has(self.sourceVolumeMode)' status: description: status represents the current information of a snapshot. properties: creationTime: - description: creationTime is the timestamp when the point-in-time - snapshot is taken by the underlying storage system. In dynamic snapshot - creation case, this field will be filled in by the CSI snapshotter - sidecar with the "creation_time" value returned from CSI "CreateSnapshot" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "creation_time" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. If not specified, it indicates - the creation time is unknown. The format of this field is a Unix - nanoseconds time encoded as an int64. On Unix, the command `date - +%s%N` returns the current time in nanoseconds since 1970-01-01 - 00:00:00 UTC. + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it indicates the creation time is unknown. + The format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. format: int64 type: integer error: - description: error is the last observed error during snapshot creation, - if any. Upon success after retry, this error field will be cleared. + description: |- + error is the last observed error during snapshot creation, if any. + Upon success after retry, this error field will be cleared. properties: message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be - logged, and it should not contain sensitive information.' + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. type: string time: description: time is the timestamp when the error was encountered. @@ -211,38 +265,40 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if a snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in by the CSI snapshotter sidecar with the "ready_to_use" - value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing - snapshot, this field will be filled with the "ready_to_use" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it, otherwise, this field will be set to "True". If not specified, - it means the readiness of a snapshot is unknown. + description: |- + readyToUse indicates if a snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. type: boolean restoreSize: - description: restoreSize represents the complete size of the snapshot - in bytes. In dynamic snapshot creation case, this field will be - filled in by the CSI snapshotter sidecar with the "size_bytes" value - returned from CSI "CreateSnapshot" gRPC call. For a pre-existing - snapshot, this field will be filled with the "size_bytes" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it. When restoring a volume from this snapshot, the size of the - volume MUST NOT be smaller than the restoreSize if it is specified, - otherwise the restoration will fail. If not specified, it indicates - that the size is unknown. + description: |- + restoreSize represents the complete size of the snapshot in bytes. + In dynamic snapshot creation case, this field will be filled in by the + CSI snapshotter sidecar with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. format: int64 minimum: 0 type: integer snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot - on the underlying storage system. If not specified, it indicates - that dynamic snapshot creation has either failed or it is still - in progress. + description: |- + snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. + If not specified, it indicates that dynamic snapshot creation has either failed + or it is still in progress. type: string - volumeGroupSnapshotContentName: - description: VolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent - of which this VolumeSnapshotContent is a part of. + volumeGroupSnapshotHandle: + description: |- + VolumeGroupSnapshotHandle is the CSI "group_snapshot_id" of a group snapshot + on the underlying storage system. type: string type: object required: diff --git a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshots.yaml b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshots.yaml index 3e7f99663..6b96d7082 100644 --- a/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshots.yaml +++ b/internal/constellation/helm/charts/edgeless/csi/charts/snapshot-crds/templates/volumesnapshots.yaml @@ -3,9 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 + controller-gen.kubebuilder.io/version: v0.15.0 api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814" - creationTimestamp: null name: volumesnapshots.snapshot.storage.k8s.io spec: group: snapshot.storage.k8s.io @@ -61,103 +60,140 @@ spec: name: v1 schema: openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time + description: |- + VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string + metadata: + type: object spec: - description: 'spec defines the desired characteristics of a snapshot requested - by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots - Required.' + description: |- + spec defines the desired characteristics of a snapshot requested by a user. + More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required. properties: source: - description: source specifies where a snapshot will be created from. - This field is immutable after creation. Required. + description: |- + source specifies where a snapshot will be created from. + This field is immutable after creation. + Required. properties: persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the - PersistentVolumeClaim object representing the volume from which - a snapshot should be created. This PVC is assumed to be in the - same namespace as the VolumeSnapshot object. This field should - be set if the snapshot does not exists, and needs to be created. + description: |- + persistentVolumeClaimName specifies the name of the PersistentVolumeClaim + object representing the volume from which a snapshot should be created. + This PVC is assumed to be in the same namespace as the VolumeSnapshot + object. + This field should be set if the snapshot does not exists, and needs to be + created. This field is immutable. type: string + x-kubernetes-validations: + - message: persistentVolumeClaimName is immutable + rule: self == oldSelf volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a - pre-existing VolumeSnapshotContent object representing an existing - volume snapshot. This field should be set if the snapshot already - exists and only needs a representation in Kubernetes. This field - is immutable. + description: |- + volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent + object representing an existing volume snapshot. + This field should be set if the snapshot already exists and only needs a representation in Kubernetes. + This field is immutable. type: string + x-kubernetes-validations: + - message: volumeSnapshotContentName is immutable + rule: self == oldSelf type: object - oneOf: - - required: ["persistentVolumeClaimName"] - - required: ["volumeSnapshotContentName"] + x-kubernetes-validations: + - message: persistentVolumeClaimName is required once set + rule: '!has(oldSelf.persistentVolumeClaimName) || has(self.persistentVolumeClaimName)' + - message: volumeSnapshotContentName is required once set + rule: '!has(oldSelf.volumeSnapshotContentName) || has(self.volumeSnapshotContentName)' + - message: exactly one of volumeSnapshotContentName and persistentVolumeClaimName + must be set + rule: (has(self.volumeSnapshotContentName) && !has(self.persistentVolumeClaimName)) + || (!has(self.volumeSnapshotContentName) && has(self.persistentVolumeClaimName)) volumeSnapshotClassName: - description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass - requested by the VolumeSnapshot. VolumeSnapshotClassName may be - left nil to indicate that the default SnapshotClass should be used. - A given cluster may have multiple default Volume SnapshotClasses: - one default per CSI Driver. If a VolumeSnapshot does not specify - a SnapshotClass, VolumeSnapshotSource will be checked to figure - out what the associated CSI Driver is, and the default VolumeSnapshotClass - associated with that CSI Driver will be used. If more than one VolumeSnapshotClass - exist for a given CSI Driver and more than one have been marked - as default, CreateSnapshot will fail and generate an event. Empty - string is not allowed for this field.' + description: |- + VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. + VolumeSnapshotClassName may be left nil to indicate that the default + SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: one + default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, + VolumeSnapshotSource will be checked to figure out what the associated + CSI Driver is, and the default VolumeSnapshotClass associated with that + CSI Driver will be used. If more than one VolumeSnapshotClass exist for + a given CSI Driver and more than one have been marked as default, + CreateSnapshot will fail and generate an event. + Empty string is not allowed for this field. type: string + x-kubernetes-validations: + - message: volumeSnapshotClassName must not be the empty string when + set + rule: size(self) > 0 required: - source type: object status: - description: status represents the current information of a snapshot. - Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent - objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent - point at each other) before using this object. + description: |- + status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and + VolumeSnapshotContent objects is successful (by validating that both + VolumeSnapshot and VolumeSnapshotContent point at each other) before + using this object. properties: boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent - object to which this VolumeSnapshot object intends to bind to. If - not specified, it indicates that the VolumeSnapshot object has not - been successfully bound to a VolumeSnapshotContent object yet. NOTE: - To avoid possible security issues, consumers must verify binding - between VolumeSnapshot and VolumeSnapshotContent objects is successful - (by validating that both VolumeSnapshot and VolumeSnapshotContent - point at each other) before using this object.' + description: |- + boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. + If not specified, it indicates that the VolumeSnapshot object has not been + successfully bound to a VolumeSnapshotContent object yet. + NOTE: To avoid possible security issues, consumers must verify binding between + VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that + both VolumeSnapshot and VolumeSnapshotContent point at each other) before using + this object. type: string creationTime: - description: creationTime is the timestamp when the point-in-time - snapshot is taken by the underlying storage system. In dynamic snapshot - creation case, this field will be filled in by the snapshot controller - with the "creation_time" value returned from CSI "CreateSnapshot" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "creation_time" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. If not specified, it may indicate - that the creation time of the snapshot is unknown. + description: |- + creationTime is the timestamp when the point-in-time snapshot is taken + by the underlying storage system. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "creation_time" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + If not specified, it may indicate that the creation time of the snapshot is unknown. format: date-time type: string error: - description: error is the last observed error during snapshot creation, - if any. This field could be helpful to upper level controllers(i.e., - application controller) to decide whether they should continue on - waiting for the snapshot to be created based on the type of error - reported. The snapshot controller will keep retrying when an error - occurs during the snapshot creation. Upon success, this error field - will be cleared. + description: |- + error is the last observed error during snapshot creation, if any. + This field could be helpful to upper level controllers(i.e., application controller) + to decide whether they should continue on waiting for the snapshot to be created + based on the type of error reported. + The snapshot controller will keep retrying when an error occurs during the + snapshot creation. Upon success, this error field will be cleared. properties: message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be - logged, and it should not contain sensitive information.' + description: |- + message is a string detailing the encountered error during snapshot + creation if specified. + NOTE: message may be logged, and it should not contain sensitive + information. type: string time: description: time is the timestamp when the error was encountered. @@ -165,32 +201,35 @@ spec: type: string type: object readyToUse: - description: readyToUse indicates if the snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in by the snapshot controller with the "ready_to_use" - value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing - snapshot, this field will be filled with the "ready_to_use" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it, otherwise, this field will be set to "True". If not specified, - it means the readiness of a snapshot is unknown. + description: |- + readyToUse indicates if the snapshot is ready to be used to restore a volume. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "ready_to_use" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "ready_to_use" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". + If not specified, it means the readiness of a snapshot is unknown. type: boolean restoreSize: type: string - description: restoreSize represents the minimum size of volume required - to create a volume from this snapshot. In dynamic snapshot creation - case, this field will be filled in by the snapshot controller with - the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. - For a pre-existing snapshot, this field will be filled with the - "size_bytes" value returned from the CSI "ListSnapshots" gRPC call - if the driver supports it. When restoring a volume from this snapshot, - the size of the volume MUST NOT be smaller than the restoreSize - if it is specified, otherwise the restoration will fail. If not - specified, it indicates that the size is unknown. + description: |- + restoreSize represents the minimum size of volume required to create a volume + from this snapshot. + In dynamic snapshot creation case, this field will be filled in by the + snapshot controller with the "size_bytes" value returned from CSI + "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the "size_bytes" + value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. + When restoring a volume from this snapshot, the size of the volume MUST NOT + be smaller than the restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true volumeGroupSnapshotName: - description: VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot - of which this VolumeSnapshot is a part of. + description: |- + VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this + VolumeSnapshot is a part of. type: string type: object required: diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml index 18dce5e37..9156e3e71 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: autoscalingstrategies.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: autoscalingstrategies.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -20,14 +21,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -48,8 +54,8 @@ spec: deployment. type: string enabled: - description: Enabled defines whether cluster autoscaling should be enabled - or not. + description: Enabled defines whether cluster autoscaling should be + enabled or not. type: boolean required: - deploymentName @@ -64,7 +70,8 @@ spec: enabled or not. type: boolean replicas: - description: Replicas is the number of replicas for the autoscaler deployment. + description: Replicas is the number of replicas for the autoscaler + deployment. format: int32 type: integer type: object @@ -73,9 +80,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml index 88fb65ae8..1beca7221 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: joiningnodes.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: joiningnodes.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: JoiningNode is the Schema for the joiningnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -59,9 +65,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml index 9c46b695c..4b7f7b7e0 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: nodeversions.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: nodeversions.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: NodeVersion is the Schema for the nodeversions API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -60,65 +66,49 @@ spec: description: AwaitingAnnotation is a list of nodes that are waiting for the operator to annotate them. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array budget: description: Budget is the amount of extra nodes that can be created @@ -129,43 +119,35 @@ spec: description: Conditions represent the latest available observations of an object's state items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -180,10 +162,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -199,516 +177,389 @@ spec: description: Donors is a list of outdated nodes that donate labels to heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array heirs: description: Heirs is a list of nodes using the latest image that still need to inherit labels from donors. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array invalid: description: Invalid is a list of invalid nodes (nodes that cannot be processed by the operator due to missing information or transient faults). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array mints: description: Mints is a list of up to date nodes that will become heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array obsolete: description: Obsolete is a list of obsolete nodes (nodes that have been created by the operator but are no longer needed). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array outdated: description: Outdated is a list of nodes that are using an outdated image. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array pending: description: Pending is a list of pending nodes (joining or leaving the cluster). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array upToDate: description: UpToDate is a list of nodes that are using the latest image and labels. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array required: + - activeclusterversionupgrade - budget - conditions type: object diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml index 41b5a4cd7..c6cd2db6a 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: pendingnodes.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: pendingnodes.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: PendingNode is the Schema for the pendingnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -34,10 +40,11 @@ spec: description: PendingNodeSpec defines the desired state of PendingNode. properties: deadline: - description: Deadline is the deadline for reaching the goal state. Joining - nodes will be terminated if the deadline is exceeded. Leaving nodes - will remain as unschedulable to prevent data loss. If not specified, - the node may remain in the pending state indefinitely. + description: |- + Deadline is the deadline for reaching the goal state. + Joining nodes will be terminated if the deadline is exceeded. + Leaving nodes will remain as unschedulable to prevent data loss. + If not specified, the node may remain in the pending state indefinitely. format: date-time type: string goal: @@ -47,8 +54,8 @@ spec: - Leave type: string groupID: - description: ScalingGroupID is the ID of the group that this node shall - be part of. + description: ScalingGroupID is the ID of the group that this node + shall be part of. type: string nodeName: description: NodeName is the kubernetes internal name of the node. @@ -72,7 +79,8 @@ spec: - Failed type: string reachedGoal: - description: ReachedGoal is true if the node has reached the goal state. + description: ReachedGoal is true if the node has reached the goal + state. type: boolean type: object type: object @@ -80,9 +88,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml index 0e334ae29..5eed4ebc8 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: scalinggroups.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: scalinggroups.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: ScalingGroup is the Schema for the scalinggroups API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -37,16 +43,16 @@ spec: description: AutoscalerGroupName is name that is expected by the autoscaler. type: string autoscaling: - description: Autoscaling specifies wether the scaling group should automatically - scale using the cluster-autoscaler. + description: Autoscaling specifies wether the scaling group should + automatically scale using the cluster-autoscaler. type: boolean groupId: - description: GroupID is the CSP specific, canonical identifier of a - scaling group. + description: GroupID is the CSP specific, canonical identifier of + a scaling group. type: string max: - description: Max is the maximum number of autoscaled nodes in the scaling - group (used by cluster-autoscaler). + description: Max is the maximum number of autoscaled nodes in the + scaling group (used by cluster-autoscaler). format: int32 type: integer min: @@ -55,11 +61,11 @@ spec: format: int32 type: integer nodeGroupName: - description: NodeGroupName is the human friendly name of the node group - as defined in the Constellation configuration. + description: NodeGroupName is the human friendly name of the node + group as defined in the Constellation configuration. type: string nodeImage: - description: NodeImage is the name of the NodeImage resource. + description: NodeVersion is the name of the NodeVersion resource. type: string role: description: Role is the role of the nodes in the scaling group. @@ -75,44 +81,36 @@ spec: description: Conditions represent the latest available observations of an object's state. items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a foo's - current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating details - about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers of - specific condition types may define expected values and meanings - for this field, and whether the values are considered a guaranteed - API. The value should be a CamelCase string. This field may - not be empty. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ @@ -126,10 +124,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -142,8 +136,8 @@ spec: type: object type: array imageReference: - description: ImageReference is the image currently used for newly created - nodes in this scaling group. + description: ImageReference is the image currently used for newly + created nodes in this scaling group. type: string required: - conditions @@ -153,9 +147,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/deployment.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/deployment.yaml index 4d2a943c4..5afe29d9b 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/deployment.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/deployment.yaml @@ -29,26 +29,9 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ .Values.kubernetesClusterDomain }} - image: {{ .Values.controllerManager.kubeRbacProxy.image.repository }}:{{ .Values.controllerManager.kubeRbacProxy.image.tag - | default .Chart.AppVersion }} - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: {{- toYaml .Values.controllerManager.kubeRbacProxy.resources | nindent - 10 }} - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8080 - --leader-elect command: - /node-operator @@ -59,6 +42,8 @@ spec: value: {{ .Values.csp | quote }} - name: constellation-uid value: {{ .Values.constellationUID | quote }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json image: {{ .Values.controllerManager.manager.image | quote }} livenessProbe: httpGet: @@ -89,6 +74,9 @@ spec: - mountPath: /etc/gce name: gceconf readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true - mountPath: /etc/constellation-upgrade-agent.sock name: upgrade-agent-socket readOnly: true @@ -126,6 +114,10 @@ spec: name: gceconf optional: true name: gceconf + - name: gcekey + secret: + secretName: gcekey + optional: true - name: upgrade-agent-socket hostPath: path: /run/constellation-upgrade-agent.sock diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml index 45dddbdd9..0e271ff7f 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -13,6 +13,7 @@ rules: verbs: - get - list + - watch - apiGroups: - "" resources: @@ -58,6 +59,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies + - joiningnodes + - nodeversions + - pendingnodes + - scalinggroups verbs: - create - delete @@ -70,38 +75,20 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers + - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes/status + - nodeversions/status + - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -120,84 +107,6 @@ rules: - nodeversion/status verbs: - get -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status - verbs: - - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/values.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/values.yaml index a5377852d..517f13887 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/values.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/values.yaml @@ -1,15 +1,4 @@ controllerManager: - kubeRbacProxy: - image: - repository: gcr.io/kubebuilder/kube-rbac-proxy - tag: v0.14.1 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi manager: resources: limits: diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/node-maintenance-operator/templates/manager-rbac.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/node-maintenance-operator/templates/manager-rbac.yaml index 52b9e568a..c5d5a211b 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/node-maintenance-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/node-maintenance-operator/templates/manager-rbac.yaml @@ -12,7 +12,13 @@ rules: resources: - namespaces verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/.helmignore b/internal/constellation/helm/charts/yawol/.helmignore similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/konnectivity/.helmignore rename to internal/constellation/helm/charts/yawol/.helmignore diff --git a/internal/constellation/helm/charts/yawol/Chart.yaml b/internal/constellation/helm/charts/yawol/Chart.yaml new file mode 100644 index 000000000..4f270e19e --- /dev/null +++ b/internal/constellation/helm/charts/yawol/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: yawol +description: A chart to deploy the yawol loadbalancer +type: application +version: 0.0.0 +dependencies: + - name: yawol-config + version: 0.0.0 + - name: yawol-controller + version: 0.0.0 diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/.helmignore b/internal/constellation/helm/charts/yawol/charts/yawol-config/.helmignore similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/.helmignore rename to internal/constellation/helm/charts/yawol/charts/yawol-config/.helmignore diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/Chart.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-config/Chart.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/Chart.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-config/Chart.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/templates/secret.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-config/templates/secret.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/templates/secret.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-config/templates/secret.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/values.schema.json b/internal/constellation/helm/charts/yawol/charts/yawol-config/values.schema.json similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/values.schema.json rename to internal/constellation/helm/charts/yawol/charts/yawol-config/values.schema.json diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/values.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-config/values.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-config/values.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-config/values.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/Chart.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/Chart.yaml similarity index 54% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/Chart.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/Chart.yaml index cccefe85b..f4dac75bd 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/Chart.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 +appVersion: v0.20.0 description: Helm chart for yawol-controller name: yawol-controller sources: - - https://github.com/stackitcloud/yawol -version: 0.14.0 -appVersion: v0.14.0 +- https://github.com/stackitcloud/yawol +version: 0.20.0 diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/README.md b/internal/constellation/helm/charts/yawol/charts/yawol-controller/README.md similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/README.md rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/README.md diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml similarity index 98% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml index 4c2ef948e..33e8052cd 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: loadbalancermachines.yawol.stackit.cloud spec: group: yawol.stackit.cloud @@ -101,6 +100,10 @@ spec: networkID: description: NetworkID defines an openstack ID for the network. type: string + subnetworkID: + description: SubnetworkID defines an openstack ID for the + subnetwork. + type: string required: - networkID type: object diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml similarity index 93% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml index 5b8b8a322..1d18db009 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancers.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: loadbalancers.yawol.stackit.cloud spec: group: yawol.stackit.cloud @@ -132,6 +131,10 @@ spec: networkID: description: NetworkID defines an openstack ID for the network. type: string + subnetworkID: + description: SubnetworkID defines an openstack ID for the + subnetwork. + type: string required: - networkID type: object @@ -232,6 +235,11 @@ spec: enabled: description: Enabled defines if log forward is enabled type: boolean + labels: + additionalProperties: + type: string + description: Labels define extra labels for loki. + type: object lokiUrl: description: 'LokiUrl defines the loki push url (Example: http://example.com:3100/loki/api/v1/push).' @@ -275,11 +283,19 @@ spec: description: ServicePort contains information on service's port. properties: appProtocol: - description: The application protocol for this port. This field - follows standard Kubernetes label syntax. Un-prefixed names - are reserved for IANA standard service names (as per RFC-6335 - and https://www.iana.org/assignments/service-names). Non-standard - protocols should use prefixed names such as mycompany.com/my-custom-protocol. + description: "The application protocol for this port. This is + used as a hint for implementations to offer richer behavior + for protocols that they understand. This field follows standard + Kubernetes label syntax. Valid values are either: \n * Un-prefixed + protocol names - reserved for IANA standard service names + (as per RFC-6335 and https://www.iana.org/assignments/service-names). + \n * Kubernetes-defined prefixed names: * 'kubernetes.io/h2c' + - HTTP/2 over cleartext as described in https://www.rfc-editor.org/rfc/rfc7540 + * 'kubernetes.io/ws' - WebSocket over cleartext as described + in https://www.rfc-editor.org/rfc/rfc6455 * 'kubernetes.io/wss' + - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + \n * Other protocols should use implementation-defined prefixed + names such as mycompany.com/my-custom-protocol." type: string name: description: The name of this port within the service. This diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml similarity index 76% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml index 6dd2ac392..9d8bea50d 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancersets.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: loadbalancersets.yawol.stackit.cloud spec: group: yawol.stackit.cloud @@ -27,6 +26,9 @@ spec: - jsonPath: .status.readyReplicas name: READY type: string + - jsonPath: .status.conditions[?(@.type=="HasKeepalivedMaster")].status + name: HasKeepalivedMaster + type: string - jsonPath: .metadata.creationTimestamp name: AGE type: date @@ -163,6 +165,10 @@ spec: description: NetworkID defines an openstack ID for the network. type: string + subnetworkID: + description: SubnetworkID defines an openstack ID + for the subnetwork. + type: string required: - networkID type: object @@ -289,6 +295,75 @@ spec: availableReplicas: description: AvailableReplicas are the current running replicas. type: integer + conditions: + description: Conditions contains condition information for a LoadBalancerSet. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array readyReplicas: description: ReadyReplicas are the current ready replicas. type: integer diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/_helpers.tpl b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/_helpers.tpl similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/_helpers.tpl rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/_helpers.tpl diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/rbac-yawol-cloud-controller.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/rbac-yawol-cloud-controller.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/rbac-yawol-cloud-controller.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/rbac-yawol-cloud-controller.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/rbac-yawol-controller.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/rbac-yawol-controller.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/rbac-yawol-controller.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/rbac-yawol-controller.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/sa-yawol-cloud-controller.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/sa-yawol-cloud-controller.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/sa-yawol-cloud-controller.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/sa-yawol-cloud-controller.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/sa-yawol-controller.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/sa-yawol-controller.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/sa-yawol-controller.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/sa-yawol-controller.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/vpa.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/vpa.yaml similarity index 100% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/vpa.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/vpa.yaml diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-cloud-controller.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-cloud-controller.yaml similarity index 95% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-cloud-controller.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-cloud-controller.yaml index 303a77ac8..496349bd5 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-cloud-controller.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-cloud-controller.yaml @@ -61,6 +61,10 @@ spec: - name: NETWORK_ID value: {{ .Values.yawolNetworkID }} {{- end }} + {{- if .Values.yawolSubnetworkID }} + - name: SUBNETWORK_ID + value: {{ .Values.yawolSubnetworkID }} + {{- end }} {{- if .Values.yawolFlavorID }} - name: FLAVOR_ID value: {{ .Values.yawolFlavorID }} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-controller.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-controller.yaml similarity index 97% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-controller.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-controller.yaml index 55cfd3694..4c369afe6 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-controller.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-controller.yaml @@ -100,6 +100,9 @@ spec: args: - -leader-elect - -enable-loadbalancermachine-controller + {{- if .Values.yawolletRequeueTime }} + - -yawollet-requeue-time={{ .Values.yawolletRequeueTime }} + {{- end }} {{- if .Values.openstackTimeout }} - -openstack-timeout={{ .Values.openstackTimeout }} {{- end }} diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml similarity index 90% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml index 47c5b58d1..39309108a 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/templates/yawol-gardener-monitoring.yaml @@ -33,7 +33,12 @@ kind: Service metadata: name: yawol-cloud-controller namespace: {{ .Release.Namespace }} + annotations: + {{- toYaml .Values.yawolCloudController.service.annotations | nindent 4 }} labels: + {{- with .Values.yawolCloudController.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} app: kubernetes role: yawol-cloud-controller spec: @@ -118,7 +123,12 @@ kind: Service metadata: name: yawol-controller namespace: {{ .Release.Namespace }} + annotations: + {{- toYaml .Values.yawolController.service.annotations | nindent 4 }} labels: + {{- with .Values.yawolController.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} app: kubernetes role: yawol-controller spec: diff --git a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/values.yaml b/internal/constellation/helm/charts/yawol/charts/yawol-controller/values.yaml similarity index 76% rename from internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/values.yaml rename to internal/constellation/helm/charts/yawol/charts/yawol-controller/values.yaml index f7f44f554..031966967 100644 --- a/internal/constellation/helm/charts/edgeless/constellation-services/charts/yawol-controller/values.yaml +++ b/internal/constellation/helm/charts/yawol/charts/yawol-controller/values.yaml @@ -4,6 +4,7 @@ podLabels: {} featureGates: {} proxy: {} namespace: kube-system + vpa: enabled: false yawolCloudController: @@ -15,19 +16,23 @@ yawolCloudController: enabled: true gardenerMonitoringEnabled: false clusterRoleEnabled: true + service: + annotations: {} + labels: {} image: - repository: ghcr.io/stackitcloud/yawol/yawol-cloud-controller - # -- Allows you to override the yawol version in this chart. Use at your own risk. - tag: "" + repository: ghcr.io/malt3/yawol/yawol-cloud-controller + tag: "yawol-controller-0.20.0-4-g6212876@sha256:ad83538fadc5d367700f75fc71c67697338307fdd81214dfc99b4cf425b8cb30" yawolController: gardenerMonitoringEnabled: false errorBackoffBaseDelay: 5ms errorBackoffMaxDelay: 1000s + service: + annotations: {} + labels: {} image: - repository: ghcr.io/stackitcloud/yawol/yawol-controller - # -- Allows you to override the yawol version in this chart. Use at your own risk. - tag: "" + repository: ghcr.io/malt3/yawol/yawol-controller + tag: "yawol-controller-0.20.0-4-g6212876@sha256:290250a851de2cf4cb6eab2d40b36724c8321b7c3c36da80fd3e2333ed6808d0" resources: yawolCloudController: @@ -61,6 +66,7 @@ resources: #yawolClassName: debug #openstackTimeout: 20s +#yawolletRequeueTime: 60 # the name of the Kubernetes secret that contains the .openrc file contents # with the correct permissions to connect to the OpenStack API @@ -78,6 +84,12 @@ yawolFloatingID: # Placed in LoadBalancer.spec.infrastructure.networkID yawolNetworkID: +# OpenStack subnetwork ID in which the Load Balancer is placed. +# If not set, the subnetwork is chosen automatically. +# +# Placed in LoadBalancer.spec.infrastructure.subnetworkID +yawolSubnetworkID: + # default value for flavor that yawol Load Balancer instances should use # can be overridden by annotation # diff --git a/internal/constellation/helm/charts/yawol/templates/.gitkeep b/internal/constellation/helm/charts/yawol/templates/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/constellation/helm/charts/yawol/values.yaml b/internal/constellation/helm/charts/yawol/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/internal/constellation/helm/chartutil.go b/internal/constellation/helm/chartutil.go index 405b57175..1f5017519 100644 --- a/internal/constellation/helm/chartutil.go +++ b/internal/constellation/helm/chartutil.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm diff --git a/internal/constellation/helm/cilium.patch b/internal/constellation/helm/cilium.patch deleted file mode 100644 index e638c66f8..000000000 --- a/internal/constellation/helm/cilium.patch +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/install/kubernetes/cilium/templates/cilium-configmap.yaml b/install/kubernetes/cilium/templates/cilium-configmap.yaml -index 4ac3b006e3..3541e3d380 100644 ---- a/install/kubernetes/cilium/templates/cilium-configmap.yaml -+++ b/install/kubernetes/cilium/templates/cilium-configmap.yaml -@@ -608,7 +608,9 @@ data: - {{- if .Values.encryption.strictMode.enabled }} - enable-encryption-strict-mode: {{ .Values.encryption.strictMode.enabled | quote }} - -- encryption-strict-mode-cidr: {{ .Values.encryption.strictMode.cidr | quote }} -+ encryption-strict-mode-node-cidrs: {{ .Values.encryption.strictMode.nodeCIDRList | join " " | quote }} -+ -+ encryption-strict-mode-pod-cidrs: {{ .Values.encryption.strictMode.podCIDRList | join " " | quote }} - - encryption-strict-mode-allow-remote-node-identities: {{ .Values.encryption.strictMode.allowRemoteNodeIdentities | quote }} - {{- end }} -diff --git a/install/kubernetes/cilium/values.yaml b/install/kubernetes/cilium/values.yaml -index c00e9af831..4661c16f56 100644 ---- a/install/kubernetes/cilium/values.yaml -+++ b/install/kubernetes/cilium/values.yaml -@@ -794,17 +794,21 @@ encryption: - # This option is only effective when encryption.type is set to "wireguard". - nodeEncryption: false - -- # -- Configure the WireGuard Pod2Pod strict mode. -+ # -- Configure the WireGuard strict mode. - strictMode: -- # -- Enable WireGuard Pod2Pod strict mode. -+ # -- Enable WireGuard strict mode. - enabled: false -+ -+ # -- podCIDRList for the WireGuard strict mode. -+ podCIDRList: [] - -- # -- CIDR for the WireGuard Pod2Pod strict mode. -- cidr: "" -+ # -- nodeCIDRList for the WireGuard strict mode. -+ nodeCIDRList: [] - - # -- Allow dynamic lookup of remote node identities. - # This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. -- allowRemoteNodeIdentities: false -+ # This is also required when control-plane nodes are exempted from node-to-node encryption. -+ allowRemoteNodeIdentities: true - - ipsec: - # -- Name of the key file inside the Kubernetes secret configured via secretName. diff --git a/internal/constellation/helm/corednsgen/BUILD.bazel b/internal/constellation/helm/corednsgen/BUILD.bazel new file mode 100644 index 000000000..4dbc61a61 --- /dev/null +++ b/internal/constellation/helm/corednsgen/BUILD.bazel @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "corednsgen_lib", + srcs = ["corednsgen.go"], + importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/helm/corednsgen", + visibility = ["//visibility:private"], + deps = [ + "//internal/versions", + "@com_github_regclient_regclient//:regclient", + "@com_github_regclient_regclient//types/ref", + "@io_k8s_api//apps/v1:apps", + "@io_k8s_api//core/v1:core", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm", + "@io_k8s_kubernetes//cmd/kubeadm/app/images", + "@io_k8s_kubernetes//cmd/kubeadm/app/phases/addons/dns", + "@io_k8s_kubernetes//cmd/kubeadm/app/util", + "@io_k8s_sigs_yaml//:yaml", + ], +) + +go_binary( + name = "corednsgen", + embed = [":corednsgen_lib"], + visibility = ["//:__subpackages__"], +) diff --git a/internal/constellation/helm/corednsgen/corednsgen.go b/internal/constellation/helm/corednsgen/corednsgen.go new file mode 100644 index 000000000..c648cca69 --- /dev/null +++ b/internal/constellation/helm/corednsgen/corednsgen.go @@ -0,0 +1,181 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +// corednsgen synthesizes a Helm chart from the resource templates embedded in +// kubeadm and writes it to the `charts` directory underneath the current +// working directory. This removes the existing `coredns` subdirectory! +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/regclient/regclient" + "github.com/regclient/regclient/types/ref" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/images" + kubedns "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "sigs.k8s.io/yaml" +) + +const configMapName = "edg-coredns" + +var chartDir = flag.String("charts", "./charts", "target directory to create charts in") + +func main() { + flag.Parse() + + if err := os.RemoveAll(filepath.Join(*chartDir, "coredns")); err != nil { + log.Fatalf("Could not remove chart dir: %v", err) + } + + writeFileRelativeToChartDir(chartYAML(), "Chart.yaml") + writeFileRelativeToChartDir(valuesYAML(), "values.yaml") + + writeTemplate(kubedns.CoreDNSServiceAccount, "serviceaccount.yaml") + writeTemplate(kubedns.CoreDNSClusterRole, "clusterrole.yaml") + writeTemplate(kubedns.CoreDNSClusterRoleBinding, "clusterrolebinding.yaml") + writeTemplate(kubedns.CoreDNSService, "service.yaml") + + writeFileRelativeToChartDir(patchedConfigMap(), "templates", "configmap.yaml") + writeFileRelativeToChartDir(patchedDeployment(), "templates", "deployment.yaml") +} + +func chartYAML() []byte { + chart := map[string]string{ + "apiVersion": "v2", + "name": "kube-dns", + "version": "0.0.0", + } + data, err := yaml.Marshal(chart) + if err != nil { + log.Fatalf("Could not marshal Chart.yaml: %v", err) + } + return data +} + +func valuesYAML() []byte { + cfg := &kubeadm.ClusterConfiguration{ + KubernetesVersion: string(versions.Default), + ImageRepository: "registry.k8s.io", + } + img := images.GetDNSImage(cfg) + ref, err := ref.New(img) + if err != nil { + log.Fatalf("Could not parse image reference: %v", err) + } + + rc := regclient.New() + m, err := rc.ManifestGet(context.Background(), ref) + if err != nil { + log.Fatalf("Could not get image manifest: %v", err) + } + + values := map[string]string{ + "clusterIP": "10.96.0.10", + "dnsDomain": "cluster.local", + "image": fmt.Sprintf("%s/%s:%s@%s", ref.Registry, ref.Repository, ref.Tag, m.GetDescriptor().Digest.String()), + } + data, err := yaml.Marshal(values) + if err != nil { + log.Fatalf("Could not marshal values.yaml: %v", err) + } + return data +} + +// patchedConfigMap renames the CoreDNS ConfigMap such that kubeadm does not find it. +// +// See https://github.com/kubernetes/kubeadm/issues/2846#issuecomment-1899942683. +func patchedConfigMap() []byte { + var cm corev1.ConfigMap + if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSConfigMap), &cm); err != nil { + log.Fatalf("Could not parse configmap: %v", err) + } + + cm.Name = configMapName + + out, err := yaml.Marshal(cm) + if err != nil { + log.Fatalf("Could not marshal patched deployment: %v", err) + } + return out +} + +// patchedDeployment extracts the CoreDNS Deployment from kubeadm, adds necessary tolerations and updates the ConfigMap reference. +func patchedDeployment() []byte { + var d appsv1.Deployment + if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSDeployment), &d); err != nil { + log.Fatalf("Could not parse deployment: %v", err) + } + + tolerations := []corev1.Toleration{ + {Key: "node.cloudprovider.kubernetes.io/uninitialized", Value: "true", Effect: corev1.TaintEffectNoSchedule}, + {Key: "node.kubernetes.io/unreachable", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, TolerationSeconds: toPtr(int64(10))}, + } + d.Spec.Template.Spec.Tolerations = append(d.Spec.Template.Spec.Tolerations, tolerations...) + + for i, vol := range d.Spec.Template.Spec.Volumes { + if vol.ConfigMap != nil { + vol.ConfigMap.Name = configMapName + } + d.Spec.Template.Spec.Volumes[i] = vol + } + + out, err := yaml.Marshal(d) + if err != nil { + log.Fatalf("Could not marshal patched deployment: %v", err) + } + return out +} + +func writeFileRelativeToChartDir(content []byte, pathElements ...string) { + p := filepath.Join(append([]string{*chartDir, "coredns"}, pathElements...)...) + d := filepath.Dir(p) + if err := os.MkdirAll(d, 0o755); err != nil { + log.Fatalf("Could not create dir %q: %v", d, err) + } + if err := os.WriteFile(p, content, 0o644); err != nil { + log.Fatalf("Could not write file %q: %v", p, err) + } +} + +// parseTemplate replaces the Go template placeholders in kubeadm resources +// with fixed values or Helm value placeholders. +func parseTemplate(tmpl string) []byte { + vars := struct { + DeploymentName, Image, ControlPlaneTaintKey, DNSDomain, DNSIP string + Replicas *int32 + }{ + DeploymentName: "coredns", + DNSDomain: `{{ .Values.dnsDomain }}`, + DNSIP: `"{{ .Values.clusterIP }}"`, + Image: `"{{ .Values.image }}"`, + ControlPlaneTaintKey: "node-role.kubernetes.io/control-plane", + Replicas: toPtr(int32(2)), + } + data, err := kubeadmutil.ParseTemplate(tmpl, vars) + if err != nil { + log.Fatalf("Could not interpolate template: %v", err) + } + return data +} + +func writeTemplate(tmpl string, name string) { + data := parseTemplate(tmpl) + writeFileRelativeToChartDir(data, "templates", name) +} + +func toPtr[T any](v T) *T { + return &v +} diff --git a/internal/constellation/helm/generateCertManager.sh b/internal/constellation/helm/generateCertManager.sh index 85cc9e672..80df82775 100755 --- a/internal/constellation/helm/generateCertManager.sh +++ b/internal/constellation/helm/generateCertManager.sh @@ -5,10 +5,10 @@ set -o errtrace shopt -s inherit_errexit echo "Pulling cert-manager Helm chart..." -version="1.12.6" +version="1.15.0" function cleanup { - rm -r "charts/cert-manager/README.md" "charts/cert-manager-v${version}.tgz" + rm -rf "charts/cert-manager/README.md" "charts/cert-manager-v${version}.tgz" } trap cleanup EXIT @@ -38,7 +38,7 @@ yq eval -i '.cainjector.image.digest = "sha256:'"${v}"'"' charts/cert-manager/va v=$(get_sha256_hash "cert-manager-acmesolver") yq eval -i '.acmesolver.image.digest = "sha256:'"${v}"'"' charts/cert-manager/values.yaml -v=$(get_sha256_hash "cert-manager-ctl") +v=$(get_sha256_hash "cert-manager-startupapicheck") yq eval -i '.startupapicheck.image.digest = "sha256:'"${v}"'"' charts/cert-manager/values.yaml echo # final newline diff --git a/internal/constellation/helm/generateCilium.sh b/internal/constellation/helm/generateCilium.sh index 9f2ac1e12..acf28ca77 100755 --- a/internal/constellation/helm/generateCilium.sh +++ b/internal/constellation/helm/generateCilium.sh @@ -7,7 +7,7 @@ shopt -s inherit_errexit echo "Pulling Cilium Helm chart..." function cleanup { - rm -r "${ciliumTmpDir}" + rm -rf -- "${ciliumTmpDir}" } trap cleanup EXIT @@ -21,14 +21,13 @@ git clone \ --no-checkout \ --sparse \ --depth 1 \ - -b 1.15.0-pre.2 \ - https://github.com/cilium/cilium.git + -b v1.15.19-edg.0 \ + https://github.com/edgelesssys/cilium.git cd cilium git sparse-checkout add install/kubernetes/cilium git checkout -git apply "${calldir}/cilium.patch" cp -r install/kubernetes/cilium "${calldir}/charts" echo # final newline diff --git a/internal/constellation/helm/helm.go b/internal/constellation/helm/helm.go index 990ec2b2b..3ac7be9e1 100644 --- a/internal/constellation/helm/helm.go +++ b/internal/constellation/helm/helm.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -35,7 +35,6 @@ import ( "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constellation/state" "github.com/edgelesssys/constellation/v2/internal/file" @@ -53,7 +52,7 @@ const ( ) type debugLog interface { - Debugf(format string, args ...any) + Debug(msg string, args ...any) } // Client is a Helm client to apply charts. @@ -91,18 +90,20 @@ type Options struct { MicroserviceVersion semver.Semver HelmWaitMode WaitMode ApplyTimeout time.Duration + OpenStackValues *OpenStackValues + ServiceCIDR string } // PrepareApply loads the charts and returns the executor to apply them. func (h Client) PrepareApply( - flags Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, openStackCfg *config.OpenStackConfig, + flags Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) (Applier, bool, error) { - releases, err := h.loadReleases(flags.CSP, flags.AttestationVariant, flags.K8sVersion, masterSecret, stateFile, flags, serviceAccURI, openStackCfg) + releases, err := h.loadReleases(masterSecret, stateFile, flags, serviceAccURI) if err != nil { return nil, false, fmt.Errorf("loading Helm releases: %w", err) } - h.log.Debugf("Loaded Helm releases") + h.log.Debug("Loaded Helm releases") actions, includesUpgrades, err := h.factory.GetActions( releases, flags.MicroserviceVersion, flags.Force, flags.AllowDestructive, flags.ApplyTimeout, ) @@ -110,12 +111,12 @@ func (h Client) PrepareApply( } func (h Client) loadReleases( - csp cloudprovider.Provider, attestationVariant variant.Variant, k8sVersion versions.ValidK8sVersion, secret uri.MasterSecret, - stateFile *state.State, flags Options, serviceAccURI string, openStackCfg *config.OpenStackConfig, + secret uri.MasterSecret, stateFile *state.State, flags Options, serviceAccURI string, ) ([]release, error) { - helmLoader := newLoader(csp, attestationVariant, k8sVersion, stateFile, h.cliVersion) - h.log.Debugf("Created new Helm loader") - return helmLoader.loadReleases(flags.Conformance, flags.DeployCSIDriver, flags.HelmWaitMode, secret, serviceAccURI, openStackCfg) + helmLoader := newLoader(flags.CSP, flags.AttestationVariant, flags.K8sVersion, stateFile, h.cliVersion) + h.log.Debug("Created new Helm loader") + // TODO(burgerdev): pass down the entire flags struct + return helmLoader.loadReleases(flags.Conformance, flags.DeployCSIDriver, flags.HelmWaitMode, secret, serviceAccURI, flags.OpenStackValues, flags.ServiceCIDR) } // Applier runs the Helm actions. @@ -133,7 +134,7 @@ type ChartApplyExecutor struct { // Apply applies the charts in order. func (c ChartApplyExecutor) Apply(ctx context.Context) error { for _, action := range c.actions { - c.log.Debugf("Applying %q", action.ReleaseName()) + c.log.Debug(fmt.Sprintf("Applying %q", action.ReleaseName())) if err := action.Apply(ctx); err != nil { return fmt.Errorf("applying %s: %w", action.ReleaseName(), err) } diff --git a/internal/constellation/helm/helm_test.go b/internal/constellation/helm/helm_test.go index 3b30de913..8b36ea80c 100644 --- a/internal/constellation/helm/helm_test.go +++ b/internal/constellation/helm/helm_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm @@ -194,11 +194,12 @@ func TestHelmApply(t *testing.T) { awsLbVersion = *tc.clusterAWSLBVersion } - certManagerVersion := "v1.12.6" // current version + certManagerVersion := "v1.15.0" // current version if tc.clusterCertManagerVersion != nil { certManagerVersion = *tc.clusterCertManagerVersion } - helmListVersion(lister, "cilium", "v1.15.0-pre.2") + helmListVersion(lister, "cilium", "v1.15.19-edg.0") + helmListVersion(lister, "coredns", "v0.0.0") helmListVersion(lister, "cert-manager", certManagerVersion) helmListVersion(lister, "constellation-services", tc.clusterMicroServiceVersion) helmListVersion(lister, "constellation-operators", tc.clusterMicroServiceVersion) @@ -217,7 +218,7 @@ func TestHelmApply(t *testing.T) { SetInfrastructure(state.Infrastructure{UID: "testuid"}). SetClusterValues(state.ClusterValues{MeasurementSalt: []byte{0x41}}), fakeServiceAccURI(csp), - uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")}, nil) + uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")}) var upgradeErr *compatibility.InvalidUpgradeError if tc.expectError { assert.Error(t, err) diff --git a/internal/constellation/helm/imageversion/BUILD.bazel b/internal/constellation/helm/imageversion/BUILD.bazel index 604bec4a2..710638a33 100644 --- a/internal/constellation/helm/imageversion/BUILD.bazel +++ b/internal/constellation/helm/imageversion/BUILD.bazel @@ -33,7 +33,5 @@ go_library( # TODO(malt3): add missing third-party images # - logstash # - filebeat -# - konnectivity-agent -# - konnectivity-server # - node-maintenance-operator # - gcp-guest-agent diff --git a/internal/constellation/helm/imageversion/imageversion.go b/internal/constellation/helm/imageversion/imageversion.go index a29126f87..266840bda 100644 --- a/internal/constellation/helm/imageversion/imageversion.go +++ b/internal/constellation/helm/imageversion/imageversion.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package imageversion contains the pinned container images for the helm charts. diff --git a/internal/constellation/helm/imageversion/placeholder.go b/internal/constellation/helm/imageversion/placeholder.go index 22bb65b41..a754ade5e 100644 --- a/internal/constellation/helm/imageversion/placeholder.go +++ b/internal/constellation/helm/imageversion/placeholder.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package imageversion diff --git a/internal/constellation/helm/loader.go b/internal/constellation/helm/loader.go index ed312cca4..61822da50 100644 --- a/internal/constellation/helm/loader.go +++ b/internal/constellation/helm/loader.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm @@ -15,13 +15,12 @@ import ( "strings" "github.com/pkg/errors" - "helm.sh/helm/pkg/ignore" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/ignore" "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constellation/helm/imageversion" "github.com/edgelesssys/constellation/v2/internal/constellation/state" @@ -32,6 +31,7 @@ import ( // Run `go generate` to download (and patch) upstream helm charts. //go:generate ./generateCilium.sh +//go:generate go run ./corednsgen/ //go:generate ./update-csi-charts.sh //go:generate ./generateCertManager.sh //go:generate ./update-aws-load-balancer-chart.sh @@ -47,6 +47,7 @@ type chartInfo struct { var ( // Charts we fetch from an upstream with real versions. + coreDNSInfo = chartInfo{releaseName: "coredns", chartName: "coredns", path: "charts/coredns"} ciliumInfo = chartInfo{releaseName: "cilium", chartName: "cilium", path: "charts/cilium"} certManagerInfo = chartInfo{releaseName: "cert-manager", chartName: "cert-manager", path: "charts/cert-manager"} awsLBControllerInfo = chartInfo{releaseName: "aws-load-balancer-controller", chartName: "aws-load-balancer-controller", path: "charts/aws-load-balancer-controller"} @@ -55,6 +56,7 @@ var ( constellationOperatorsInfo = chartInfo{releaseName: "constellation-operators", chartName: "constellation-operators", path: "charts/edgeless/operators"} constellationServicesInfo = chartInfo{releaseName: "constellation-services", chartName: "constellation-services", path: "charts/edgeless/constellation-services"} csiInfo = chartInfo{releaseName: "constellation-csi", chartName: "constellation-csi", path: "charts/edgeless/csi"} + yawolLBControllerInfo = chartInfo{releaseName: "yawol", chartName: "yawol", path: "charts/yawol"} ) // chartLoader loads embedded helm charts. @@ -68,7 +70,6 @@ type chartLoader struct { autoscalerImage string verificationServiceImage string gcpGuestAgentImage string - konnectivityImage string constellationOperatorImage string nodeMaintenanceOperatorImage string clusterName string @@ -104,7 +105,6 @@ func newLoader(csp cloudprovider.Provider, attestationVariant variant.Variant, k autoscalerImage: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage, verificationServiceImage: imageversion.VerificationService("", ""), gcpGuestAgentImage: versions.GcpGuestImage, - konnectivityImage: versions.KonnectivityAgentImage, constellationOperatorImage: imageversion.ConstellationNodeOperator("", ""), nodeMaintenanceOperatorImage: versions.NodeMaintenanceOperatorImage, } @@ -116,9 +116,17 @@ func newLoader(csp cloudprovider.Provider, attestationVariant variant.Variant, k // that the new release is installed after the existing one to avoid name conflicts. type releaseApplyOrder []release +// OpenStackValues are helm values for OpenStack. +type OpenStackValues struct { + DeployYawolLoadBalancer bool + FloatingIPPoolID string + YawolFlavorID string + YawolImageID string +} + // loadReleases loads the embedded helm charts and returns them as a HelmReleases object. func (i *chartLoader) loadReleases(conformanceMode, deployCSIDriver bool, helmWaitMode WaitMode, masterSecret uri.MasterSecret, - serviceAccURI string, openStackCfg *config.OpenStackConfig, + serviceAccURI string, openStackValues *OpenStackValues, serviceCIDR string, ) (releaseApplyOrder, error) { ciliumRelease, err := i.loadRelease(ciliumInfo, helmWaitMode) if err != nil { @@ -127,6 +135,16 @@ func (i *chartLoader) loadReleases(conformanceMode, deployCSIDriver bool, helmWa ciliumVals := extraCiliumValues(i.csp, conformanceMode, i.stateFile.Infrastructure) ciliumRelease.values = mergeMaps(ciliumRelease.values, ciliumVals) + coreDNSRelease, err := i.loadRelease(coreDNSInfo, helmWaitMode) + if err != nil { + return nil, fmt.Errorf("loading coredns: %w", err) + } + coreDNSVals, err := extraCoreDNSValues(serviceCIDR) + if err != nil { + return nil, fmt.Errorf("loading coredns values: %w", err) + } + coreDNSRelease.values = mergeMaps(coreDNSRelease.values, coreDNSVals) + certManagerRelease, err := i.loadRelease(certManagerInfo, helmWaitMode) if err != nil { return nil, fmt.Errorf("loading cert-manager: %w", err) @@ -144,15 +162,15 @@ func (i *chartLoader) loadReleases(conformanceMode, deployCSIDriver bool, helmWa } svcVals, err := extraConstellationServicesValues(i.csp, i.attestationVariant, masterSecret, - serviceAccURI, i.stateFile.Infrastructure, openStackCfg) + serviceAccURI, i.stateFile.Infrastructure, openStackValues) if err != nil { return nil, fmt.Errorf("extending constellation-services values: %w", err) } conServicesRelease.values = mergeMaps(conServicesRelease.values, svcVals) - releases := releaseApplyOrder{ciliumRelease, conServicesRelease, certManagerRelease} + releases := releaseApplyOrder{ciliumRelease, coreDNSRelease, conServicesRelease, certManagerRelease, operatorRelease} if deployCSIDriver { - csiRelease, err := i.loadRelease(csiInfo, helmWaitMode) + csiRelease, err := i.loadRelease(csiInfo, WaitModeNone) if err != nil { return nil, fmt.Errorf("loading snapshot CRDs: %w", err) } @@ -164,13 +182,30 @@ func (i *chartLoader) loadReleases(conformanceMode, deployCSIDriver bool, helmWa releases = append(releases, csiRelease) } if i.csp == cloudprovider.AWS { - awsRelease, err := i.loadRelease(awsLBControllerInfo, helmWaitMode) + awsRelease, err := i.loadRelease(awsLBControllerInfo, WaitModeNone) if err != nil { return nil, fmt.Errorf("loading aws-services: %w", err) } releases = append(releases, awsRelease) } - releases = append(releases, operatorRelease) + if i.csp == cloudprovider.OpenStack { + if openStackValues == nil { + return nil, errors.New("provider is OpenStack but OpenStack config is missing") + } + if openStackValues.DeployYawolLoadBalancer { + yawolRelease, err := i.loadRelease(yawolLBControllerInfo, WaitModeNone) + if err != nil { + return nil, fmt.Errorf("loading yawol chart: %w", err) + } + + yawolVals, err := extraYawolValues(serviceAccURI, i.stateFile.Infrastructure, openStackValues) + if err != nil { + return nil, fmt.Errorf("extending yawol chart values: %w", err) + } + yawolRelease.values = mergeMaps(yawolRelease.values, yawolVals) + releases = append(releases, yawolRelease) + } + } return releases, nil } @@ -187,10 +222,9 @@ func (i *chartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (releas switch info.releaseName { case ciliumInfo.releaseName: - var ok bool - values, ok = ciliumVals[i.csp.String()] - if !ok { - return release{}, fmt.Errorf("cilium values for csp %q not found", i.csp.String()) + values, err = i.loadCiliumValues(i.csp) + if err != nil { + return release{}, fmt.Errorf("loading cilium values: %w", err) } case certManagerInfo.releaseName: values = i.loadCertManagerValues() @@ -202,6 +236,8 @@ func (i *chartLoader) loadRelease(info chartInfo, helmWaitMode WaitMode) (releas values = i.loadAWSLBControllerValues() case csiInfo.releaseName: values = i.loadCSIValues() + default: + values = map[string]any{} } // Charts we package ourselves have version 0.0.0. @@ -232,9 +268,17 @@ func (i *chartLoader) loadCertManagerValues() map[string]any { "tolerations": controlPlaneTolerations, "webhook": map[string]any{ "tolerations": controlPlaneTolerations, + "podDisruptionBudget": map[string]any{ + "enabled": true, + }, + "replicaCount": 2, }, "cainjector": map[string]any{ "tolerations": controlPlaneTolerations, + "podDisruptionBudget": map[string]any{ + "enabled": true, + }, + "replicaCount": 2, }, "startupapicheck": map[string]any{ "timeout": "5m", @@ -243,6 +287,10 @@ func (i *chartLoader) loadCertManagerValues() map[string]any { }, "tolerations": controlPlaneTolerations, }, + "podDisruptionBudget": map[string]any{ + "enabled": true, + }, + "replicaCount": 2, } } @@ -307,9 +355,6 @@ func (i *chartLoader) loadConstellationServicesValues() map[string]any { "gcp-guest-agent": map[string]any{ "image": i.gcpGuestAgentImage, }, - "konnectivity": map[string]any{ - "image": i.konnectivityImage, - }, "tags": i.cspTags(), } } @@ -326,6 +371,92 @@ func (i *chartLoader) cspTags() map[string]any { } } +func (i *chartLoader) loadCiliumValues(cloudprovider.Provider) (map[string]any, error) { + sharedConfig := map[string]any{ + "extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label", "--bpf-filter-priority=128"}, + "endpointRoutes": map[string]any{ + "enabled": true, + }, + "l7Proxy": false, + "image": map[string]any{ + "repository": "ghcr.io/edgelesssys/cilium/cilium", + "suffix": "", + "tag": "v1.15.19-edg.0", + "digest": "sha256:700218a5ffc10473ce9b09d560b8e0e3ed1309a4d57a9273da2ed16e3e1533f3", + "useDigest": true, + }, + "operator": map[string]any{ + "image": map[string]any{ + "repository": "ghcr.io/edgelesssys/cilium/operator", + "suffix": "", + "tag": "v1.15.19-edg.0", + // Careful: this is the digest of ghcr.io/.../operator-generic! + // See magic image manipulation in ./helm/charts/cilium/templates/cilium-operator/_helpers.tpl. + "genericDigest": "sha256:5db046fea42cb1239d4eaa0f870d10e77911768a1eaf34c4968488dea93e27c4", + "useDigest": true, + }, + "podDisruptionBudget": map[string]any{ + "enabled": true, + }, + }, + "encryption": map[string]any{ + "enabled": true, + "type": "wireguard", + "nodeEncryption": true, + "strictMode": map[string]any{ + "enabled": true, + "podCIDRList": []string{"10.244.0.0/16"}, + "allowRemoteNodeIdentities": false, + }, + }, + "ipam": map[string]any{ + "operator": map[string]any{ + "clusterPoolIPv4PodCIDRList": []string{ + "10.244.0.0/16", + }, + }, + }, + "bpf": map[string]any{ + "masquerade": true, + }, + "ipMasqAgent": map[string]any{ + "enabled": true, + "config": map[string]any{ + "masqLinkLocal": true, + }, + }, + "kubeProxyReplacement": "strict", + "enableCiliumEndpointSlice": true, + "kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256", + "cleanBpfState": true, + } + cspOverrideConfigs := map[string]map[string]any{ + cloudprovider.AWS.String(): {}, + cloudprovider.Azure.String(): {}, + cloudprovider.GCP.String(): { + "routingMode": "native", + "encryption": map[string]any{ + "strictMode": map[string]any{ + "podCIDRList": []string{""}, + }, + }, + "ipam": map[string]any{ + "mode": "kubernetes", + }, + }, + cloudprovider.OpenStack.String(): {}, + cloudprovider.QEMU.String(): { + "extraArgs": []string{""}, + }, + } + + cspValues, ok := cspOverrideConfigs[i.csp.String()] + if !ok { + return nil, fmt.Errorf("cilium values for csp %q not found", i.csp.String()) + } + return mergeMaps(sharedConfig, cspValues), nil +} + // updateVersions changes all versions of direct dependencies that are set to "0.0.0" to newVersion. func updateVersions(chart *chart.Chart, newVersion semver.Semver) { chart.Metadata.Version = newVersion.String() diff --git a/internal/constellation/helm/loader_test.go b/internal/constellation/helm/loader_test.go index 2d069589a..6ee767e73 100644 --- a/internal/constellation/helm/loader_test.go +++ b/internal/constellation/helm/loader_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm @@ -27,6 +27,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constellation/state" "github.com/edgelesssys/constellation/v2/internal/kms/uri" @@ -58,6 +59,18 @@ func fakeServiceAccURI(provider cloudprovider.Provider) string { UamiResourceID: "uid", } return creds.ToCloudServiceAccountURI() + case cloudprovider.OpenStack: + creds := openstack.AccountKey{ + AuthURL: "authURL", + Username: "username", + Password: "password", + ProjectID: "projectID", + ProjectName: "projectName", + UserDomainName: "userDomainName", + ProjectDomainName: "projectDomainName", + RegionName: "regionName", + } + return creds.ToCloudServiceAccountURI() default: return "" } @@ -81,7 +94,7 @@ func TestLoadReleases(t *testing.T) { helmReleases, err := chartLoader.loadReleases( true, false, WaitModeAtomic, uri.MasterSecret{Key: []byte("secret"), Salt: []byte("masterSalt")}, - fakeServiceAccURI(cloudprovider.GCP), nil, + fakeServiceAccURI(cloudprovider.GCP), nil, "172.16.128.0/17", ) require.NoError(err) for _, release := range helmReleases { @@ -162,6 +175,19 @@ func TestConstellationServices(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) + var openstackValues *OpenStackValues + if tc.config.Provider.OpenStack != nil { + var deploy bool + if tc.config.Provider.OpenStack.DeployYawolLoadBalancer != nil { + deploy = *tc.config.Provider.OpenStack.DeployYawolLoadBalancer + } + openstackValues = &OpenStackValues{ + DeployYawolLoadBalancer: deploy, + FloatingIPPoolID: tc.config.Provider.OpenStack.FloatingIPPoolID, + YawolFlavorID: tc.config.Provider.OpenStack.YawolFlavorID, + YawolImageID: tc.config.Provider.OpenStack.YawolImageID, + } + } chartLoader := chartLoader{ csp: tc.config.GetProvider(), @@ -171,7 +197,6 @@ func TestConstellationServices(t *testing.T) { azureCNMImage: tc.cnmImage, autoscalerImage: "autoscalerImage", verificationServiceImage: "verificationImage", - konnectivityImage: "konnectivityImage", gcpGuestAgentImage: "gcpGuestAgentImage", clusterName: "testCluster", } @@ -187,7 +212,7 @@ func TestConstellationServices(t *testing.T) { UID: "uid", Azure: &state.Azure{}, GCP: &state.GCP{}, - }, tc.config.Provider.OpenStack) + }, openstackValues) require.NoError(err) values = mergeMaps(values, extraVals) @@ -235,6 +260,55 @@ func TestConstellationServices(t *testing.T) { } } +func TestExtraCoreDNSValues(t *testing.T) { + testCases := map[string]struct { + cidr string + wantIP string + wantUnset bool + wantErr bool + }{ + "default": { + cidr: "10.96.0.0/12", + wantIP: "10.96.0.10", + }, + "custom": { + cidr: "172.16.128.0/17", + wantIP: "172.16.128.10", + }, + "too small": { + cidr: "172.16.0.0/30", + wantErr: true, + }, + "bad ip": { + cidr: "cluster.local", + wantErr: true, + }, + "v6": { + cidr: "fd12:3456:789a:100::/56", + wantIP: "fd12:3456:789a:100::a", + }, + "no ip": { + wantUnset: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + values, err := extraCoreDNSValues(tc.cidr) + if tc.wantErr { + assert.Error(t, err) + return + } + ip, ok := values["clusterIP"] + if tc.wantUnset { + assert.False(t, ok) + return + } + assert.Equal(t, tc.wantIP, ip) + }) + } +} + // TestOperators checks if the rendered constellation-services chart produces the expected yaml files. func TestOperators(t *testing.T) { testCases := map[string]struct { @@ -384,12 +458,6 @@ func addInClusterValues(values map[string]any, csp cloudprovider.Provider) error } verificationVals["loadBalancerIP"] = "127.0.0.1" - konnectivityVals, ok := values["konnectivity"].(map[string]any) - if !ok { - return errors.New("missing 'konnectivity' key") - } - konnectivityVals["loadBalancerIP"] = "127.0.0.1" - ccmVals, ok := values["ccm"].(map[string]any) if !ok { return errors.New("missing 'ccm' key") diff --git a/internal/constellation/helm/overrides.go b/internal/constellation/helm/overrides.go index 60c8be7da..6c05c0b10 100644 --- a/internal/constellation/helm/overrides.go +++ b/internal/constellation/helm/overrides.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* Overrides contains helm values that are dynamically injected into the helm charts. @@ -18,12 +18,24 @@ import ( "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" - "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/constellation/state" "github.com/edgelesssys/constellation/v2/internal/kms/uri" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) +func extraCoreDNSValues(serviceCIDR string) (map[string]any, error) { + if serviceCIDR == "" { + return map[string]any{}, nil + } + ip, err := kubeadmconstants.GetDNSIP(serviceCIDR) + if err != nil { + return nil, fmt.Errorf("calculating DNS service IP: %w", err) + } + + return map[string]any{"clusterIP": ip.String()}, nil +} + // TODO(malt3): switch over to DNS name on AWS and Azure // soon as every apiserver certificate of every control-plane node // has the dns endpoint in its SAN list. @@ -34,17 +46,11 @@ import ( // Also, the charts are not rendered correctly without all of these values. func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, output state.Infrastructure) map[string]any { extraVals := map[string]any{} - if conformanceMode { - extraVals["kubeProxyReplacementHealthzBindAddr"] = "" - extraVals["kubeProxyReplacement"] = "partial" - extraVals["sessionAffinity"] = true - extraVals["cni"] = map[string]any{ - "chainingMode": "portmap", - } - } strictMode := map[string]any{} - if provider != cloudprovider.QEMU { + // TODO: Once we are able to set the subnet of the load balancer VMs + // on STACKIT, we can remove the OpenStack exception here. + if provider != cloudprovider.QEMU && provider != cloudprovider.OpenStack { strictMode = map[string]any{ "nodeCIDRList": []string{output.IPCidrNode}, } @@ -59,12 +65,45 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou extraVals["encryption"] = map[string]any{ "strictMode": strictMode, } + + // On QEMU e.g. the join-service must talk to our mini-qemu-metadata docker container + // This container runs inside the node CIDR, so we need to masq any pod traffic to it + // with the node's IP address. To archive that, we override Cilium's default masq ranges + // with an empty list. + masqCIDRs := []string{} + if provider != cloudprovider.QEMU { + masqCIDRs = append(masqCIDRs, output.IPCidrNode) + } extraVals["ipMasqAgent"] = map[string]any{ "config": map[string]any{ - "nonMasqueradeCIDRs": []string{output.IPCidrNode}, + "nonMasqueradeCIDRs": masqCIDRs, }, } + // When --conformance is set, we try to mitigate https://github.com/cilium/cilium/issues/9207 + // Users are discouraged of ever using this mode, except if they truly + // require protocol differentiation to work and cannot mitigate that any other way. + // Since there should always be workarounds, we only support this mode to + // pass the K8s conformance tests. It is not supported to switch to or from + // this mode after Constellation has been initialized. + if conformanceMode { + extraVals["kubeProxyReplacementHealthzBindAddr"] = "" + extraVals["kubeProxyReplacement"] = "false" + extraVals["sessionAffinity"] = true + extraVals["cni"] = map[string]any{ + "chainingMode": "portmap", + } + extraVals["ipMasqAgent"] = map[string]any{ + "enabled": false, + } + extraVals["bpf"] = map[string]any{ + "masquerade": false, + } + extraVals["k8s"] = map[string]any{ + "serviceProxyName": "cilium", + } + } + return extraVals } @@ -72,7 +111,7 @@ func extraCiliumValues(provider cloudprovider.Provider, conformanceMode bool, ou // Values set inside this function are only applied during init, not during upgrade. func extraConstellationServicesValues( csp cloudprovider.Provider, attestationVariant variant.Variant, masterSecret uri.MasterSecret, serviceAccURI string, - output state.Infrastructure, openStackCfg *config.OpenStackConfig, + output state.Infrastructure, openStackCfg *OpenStackValues, ) (map[string]any, error) { extraVals := map[string]any{} extraVals["join-service"] = map[string]any{ @@ -81,9 +120,6 @@ func extraConstellationServicesValues( extraVals["verification-service"] = map[string]any{ "attestationVariant": attestationVariant.String(), } - extraVals["konnectivity"] = map[string]any{ - "loadBalancerIP": output.ClusterEndpoint, - } extraVals["key-service"] = map[string]any{ "masterSecret": base64.StdEncoding.EncodeToString(masterSecret.Key), @@ -91,21 +127,18 @@ func extraConstellationServicesValues( } switch csp { case cloudprovider.OpenStack: + creds, err := openstack.AccountKeyFromURI(serviceAccURI) + if err != nil { + return nil, err + } + credsIni := creds.CloudINI().FullConfiguration() if openStackCfg == nil { return nil, fmt.Errorf("no OpenStack config") } - extraVals["openstack"] = map[string]any{ - "deployYawolLoadBalancer": openStackCfg.DeployYawolLoadBalancer != nil && *openStackCfg.DeployYawolLoadBalancer, - } - if openStackCfg.DeployYawolLoadBalancer != nil && *openStackCfg.DeployYawolLoadBalancer { - extraVals["yawol-controller"] = map[string]any{ - "yawolOSSecretName": "yawolkey", - // has to be larger than ~30s to account for slow OpenStack API calls. - "openstackTimeout": "1m", - "yawolFloatingID": openStackCfg.FloatingIPPoolID, - "yawolFlavorID": openStackCfg.YawolFlavorID, - "yawolImageID": openStackCfg.YawolImageID, - } + extraVals["ccm"] = map[string]any{ + "OpenStack": map[string]any{ + "secretData": credsIni, + }, } case cloudprovider.GCP: serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(serviceAccURI) @@ -145,6 +178,36 @@ func extraConstellationServicesValues( return extraVals, nil } +// extraYawolValues extends the given values map by some values depending on user input. +// Values set inside this function are only applied during init, not during upgrade. +func extraYawolValues(serviceAccURI string, output state.Infrastructure, openStackCfg *OpenStackValues) (map[string]any, error) { + extraVals := map[string]any{} + + creds, err := openstack.AccountKeyFromURI(serviceAccURI) + if err != nil { + return nil, err + } + yawolIni := creds.CloudINI().YawolConfiguration() + extraVals["yawol-config"] = map[string]any{ + "secretData": yawolIni, + } + if openStackCfg != nil && openStackCfg.DeployYawolLoadBalancer { + extraVals["yawol-controller"] = map[string]any{ + "yawolOSSecretName": "yawolkey", + // has to be larger than ~30s to account for slow OpenStack API calls. + "openstackTimeout": "1m", + "yawolFloatingID": openStackCfg.FloatingIPPoolID, + "yawolFlavorID": openStackCfg.YawolFlavorID, + "yawolImageID": openStackCfg.YawolImageID, + "yawolNetworkID": output.OpenStack.NetworkID, + "yawolSubnetworkID": output.OpenStack.SubnetID, + "yawolAPIHost": fmt.Sprintf("https://%s:%d", output.InClusterEndpoint, constants.KubernetesPort), + } + } + + return extraVals, nil +} + // cloudConfig is used to marshal the cloud config for the Kubernetes Cloud Controller Manager on Azure. type cloudConfig struct { Cloud string `json:"cloud,omitempty"` @@ -180,7 +243,7 @@ func getCCMConfig(azureState state.Azure, serviceAccURI string) ([]byte, error) ResourceGroup: azureState.ResourceGroup, LoadBalancerSku: "standard", SecurityGroupName: azureState.NetworkSecurityGroupName, - LoadBalancerName: azureState.LoadBalancerName, + LoadBalancerName: "kubernetes-lb", UseInstanceMetadata: true, VMType: "vmss", Location: creds.Location, diff --git a/internal/constellation/helm/release.go b/internal/constellation/helm/release.go index c7be7ab5c..21ad1e9aa 100644 --- a/internal/constellation/helm/release.go +++ b/internal/constellation/helm/release.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package helm provides types and functions shared across services. diff --git a/internal/constellation/helm/retryaction.go b/internal/constellation/helm/retryaction.go index ca61944b0..4725c8659 100644 --- a/internal/constellation/helm/retryaction.go +++ b/internal/constellation/helm/retryaction.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm @@ -28,7 +28,7 @@ type retrieableApplier interface { // retryApply retries the given retriable action. func retryApply(ctx context.Context, action retrieableApplier, retryInterval time.Duration, log debugLog) error { var retries int - retriable := func(err error) bool { + retriable := func(_ error) bool { // abort after maximumRetryAttempts tries. if retries >= maximumRetryAttempts { return false @@ -49,7 +49,7 @@ func retryApply(ctx context.Context, action retrieableApplier, retryInterval tim return fmt.Errorf("helm install: %w", err) } retryLoopFinishDuration := time.Since(retryLoopStartTime) - log.Debugf("Helm chart %q installation finished after %s", action.ReleaseName(), retryLoopFinishDuration) + log.Debug(fmt.Sprintf("Helm chart %q installation finished after %q", action.ReleaseName(), retryLoopFinishDuration)) return nil } @@ -61,9 +61,9 @@ type applyDoer struct { // Do tries to apply the action. func (i applyDoer) Do(ctx context.Context) error { - i.log.Debugf("Trying to apply Helm chart %s", i.applier.ReleaseName()) + i.log.Debug(fmt.Sprintf("Trying to apply Helm chart %q", i.applier.ReleaseName())) if err := i.applier.apply(ctx); err != nil { - i.log.Debugf("Helm chart installation %s failed: %v", i.applier.ReleaseName(), err) + i.log.Debug(fmt.Sprintf("Helm chart installation %q failed: %q", i.applier.ReleaseName(), err)) return err } diff --git a/internal/constellation/helm/retryaction_test.go b/internal/constellation/helm/retryaction_test.go index 6a39d7cb2..f7259a358 100644 --- a/internal/constellation/helm/retryaction_test.go +++ b/internal/constellation/helm/retryaction_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm @@ -64,7 +64,7 @@ func TestRetryApply(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - err := retryApply(context.Background(), tc.applier, time.Millisecond, logger.NewTest(t)) + err := retryApply(t.Context(), tc.applier, time.Millisecond, logger.NewTest(t)) if tc.wantErr { assert.Error(err) } else { diff --git a/internal/constellation/helm/serviceversion.go b/internal/constellation/helm/serviceversion.go index a3d9ca57c..06603e293 100644 --- a/internal/constellation/helm/serviceversion.go +++ b/internal/constellation/helm/serviceversion.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm diff --git a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/deployment.yaml b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/deployment.yaml index b8ff3204e..fa41d793b 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/deployment.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/deployment.yaml @@ -37,79 +37,63 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /node-operator - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - - name: CONSTEL_CSP - value: GCP - - name: constellation-uid - value: "42424242424242" - image: constellationOperatorImage - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /etc/kubernetes/pki/etcd - name: etcd-certs - - mountPath: /host/usr/lib/os-release - name: usr-lib-os-release - - mountPath: /etc/os-release - name: etc-os-release - - mountPath: /etc/azure - name: azureconfig - readOnly: true - - mountPath: /etc/gce - name: gceconf - readOnly: true - - mountPath: /etc/constellation-upgrade-agent.sock - name: upgrade-agent-socket - readOnly: true + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8080 + - --leader-elect + command: + - /node-operator + env: + - name: KUBERNETES_CLUSTER_DOMAIN + value: cluster.local + - name: CONSTEL_CSP + value: GCP + - name: constellation-uid + value: "42424242424242" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + image: constellationOperatorImage + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + - mountPath: /host/usr/lib/os-release + name: usr-lib-os-release + - mountPath: /etc/os-release + name: etc-os-release + - mountPath: /etc/azure + name: azureconfig + readOnly: true + - mountPath: /etc/gce + name: gceconf + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /etc/constellation-upgrade-agent.sock + name: upgrade-agent-socket + readOnly: true nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: @@ -117,34 +101,38 @@ spec: serviceAccountName: constellation-operator-controller-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists volumes: - - hostPath: - path: /etc/kubernetes/pki/etcd - type: Directory - name: etcd-certs - - hostPath: - path: /usr/lib/os-release - type: File - name: usr-lib-os-release - - hostPath: - path: /etc/os-release - type: File - name: etc-os-release - - name: azureconfig - secret: - optional: true - secretName: azureconfig - - configMap: + - hostPath: + path: /etc/kubernetes/pki/etcd + type: Directory + name: etcd-certs + - hostPath: + path: /usr/lib/os-release + type: File + name: usr-lib-os-release + - hostPath: + path: /etc/os-release + type: File + name: etc-os-release + - name: azureconfig + secret: + optional: true + secretName: azureconfig + - configMap: + name: gceconf + optional: true name: gceconf - optional: true - name: gceconf - - name: upgrade-agent-socket - hostPath: - path: /run/constellation-upgrade-agent.sock - type: Socket + - name: gcekey + secret: + secretName: gcekey + optional: true + - name: upgrade-agent-socket + hostPath: + path: /run/constellation-upgrade-agent.sock + type: Socket diff --git a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c8..56bf77080 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -16,6 +16,7 @@ rules: verbs: - get - list + - watch - apiGroups: - "" resources: @@ -61,6 +62,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies + - joiningnodes + - nodeversions + - pendingnodes + - scalinggroups verbs: - create - delete @@ -73,38 +78,20 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers + - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes/status + - nodeversions/status + - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -123,84 +110,6 @@ rules: - nodeversion/status verbs: - get -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status - verbs: - - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml index fa9f582a4..e364d1498 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml @@ -15,7 +15,13 @@ rules: resources: - namespaces verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/internal/constellation/helm/testdata/AWS/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml b/internal/constellation/helm/testdata/AWS/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml new file mode 100644 index 000000000..0cf4c3f74 --- /dev/null +++ b/internal/constellation/helm/testdata/AWS/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns-pdb + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns diff --git a/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml index 2156f82a6..538883439 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-services/charts/join-service/templates/daemonset.yaml @@ -40,6 +40,9 @@ spec: - --cloud-provider=AWS - --key-service-endpoint=key-service.testNamespace:9000 - --attestation-variant=aws-nitro-tpm + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json volumeMounts: - mountPath: /var/config name: config @@ -47,6 +50,14 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /var/run/state/ssh + name: ssh ports: - containerPort: 9090 name: tcp @@ -54,6 +65,10 @@ spec: securityContext: privileged: true volumes: + - name: gcekey + secret: + secretName: gcekey + optional: true - name: config projected: sources: @@ -64,4 +79,10 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: ssh + hostPath: + path: /var/run/state/ssh updateStrategy: {} diff --git a/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml b/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml deleted file mode 100644 index f189cb6a3..000000000 --- a/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: system:konnectivity-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:konnectivity-server diff --git a/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/daemonset.yaml b/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/daemonset.yaml deleted file mode 100644 index 0f26cfbb9..000000000 --- a/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/daemonset.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: konnectivity-agent - name: konnectivity-agent - namespace: testNamespace -spec: - selector: - matchLabels: - k8s-app: konnectivity-agent - template: - metadata: - labels: - k8s-app: konnectivity-agent - spec: - containers: - - args: - - --logtostderr=true - - --proxy-server-host=127.0.0.1 - - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --proxy-server-port=8132 - - --admin-server-port=8133 - - --health-server-port=8134 - - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token - - --agent-identifiers=host=$(HOST_IP) - - --sync-forever=true - - --keepalive-time=60m - - --sync-interval=5s - - --sync-interval-cap=30s - - --probe-interval=5s - - --v=3 - command: - - /proxy-agent - env: - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - image: konnectivityImage - livenessProbe: - httpGet: - path: /healthz - port: 8134 - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: konnectivity-agent - resources: {} - volumeMounts: - - mountPath: /var/run/secrets/tokens - name: konnectivity-agent-token - readOnly: true - priorityClassName: system-cluster-critical - serviceAccountName: konnectivity-agent - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - name: konnectivity-agent-token - projected: - sources: - - serviceAccountToken: - audience: system:konnectivity-server - path: konnectivity-agent-token - updateStrategy: {} diff --git a/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/serviceaccount.yaml b/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/serviceaccount.yaml deleted file mode 100644 index ad307c56f..000000000 --- a/internal/constellation/helm/testdata/AWS/constellation-services/charts/konnectivity/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: konnectivity-agent - namespace: testNamespace diff --git a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/deployment.yaml b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/deployment.yaml index 55e863816..23b5ac730 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/deployment.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/deployment.yaml @@ -37,79 +37,63 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /node-operator - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - - name: CONSTEL_CSP - value: Azure - - name: constellation-uid - value: "42424242424242" - image: constellationOperatorImage - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /etc/kubernetes/pki/etcd - name: etcd-certs - - mountPath: /host/usr/lib/os-release - name: usr-lib-os-release - - mountPath: /etc/os-release - name: etc-os-release - - mountPath: /etc/azure - name: azureconfig - readOnly: true - - mountPath: /etc/gce - name: gceconf - readOnly: true - - mountPath: /etc/constellation-upgrade-agent.sock - name: upgrade-agent-socket - readOnly: true + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8080 + - --leader-elect + command: + - /node-operator + env: + - name: KUBERNETES_CLUSTER_DOMAIN + value: cluster.local + - name: CONSTEL_CSP + value: Azure + - name: constellation-uid + value: "42424242424242" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + image: constellationOperatorImage + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + - mountPath: /host/usr/lib/os-release + name: usr-lib-os-release + - mountPath: /etc/os-release + name: etc-os-release + - mountPath: /etc/azure + name: azureconfig + readOnly: true + - mountPath: /etc/gce + name: gceconf + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /etc/constellation-upgrade-agent.sock + name: upgrade-agent-socket + readOnly: true nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: @@ -117,34 +101,38 @@ spec: serviceAccountName: constellation-operator-controller-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists volumes: - - hostPath: - path: /etc/kubernetes/pki/etcd - type: Directory - name: etcd-certs - - hostPath: - path: /usr/lib/os-release - type: File - name: usr-lib-os-release - - hostPath: - path: /etc/os-release - type: File - name: etc-os-release - - name: azureconfig - secret: - optional: true - secretName: azureconfig - - configMap: + - hostPath: + path: /etc/kubernetes/pki/etcd + type: Directory + name: etcd-certs + - hostPath: + path: /usr/lib/os-release + type: File + name: usr-lib-os-release + - hostPath: + path: /etc/os-release + type: File + name: etc-os-release + - name: azureconfig + secret: + optional: true + secretName: azureconfig + - configMap: + name: gceconf + optional: true name: gceconf - optional: true - name: gceconf - - name: upgrade-agent-socket - hostPath: - path: /run/constellation-upgrade-agent.sock - type: Socket + - name: gcekey + secret: + secretName: gcekey + optional: true + - name: upgrade-agent-socket + hostPath: + path: /run/constellation-upgrade-agent.sock + type: Socket diff --git a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c8..56bf77080 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -16,6 +16,7 @@ rules: verbs: - get - list + - watch - apiGroups: - "" resources: @@ -61,6 +62,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies + - joiningnodes + - nodeversions + - pendingnodes + - scalinggroups verbs: - create - delete @@ -73,38 +78,20 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers + - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes/status + - nodeversions/status + - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -123,84 +110,6 @@ rules: - nodeversion/status verbs: - get -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status - verbs: - - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml index fa9f582a4..e364d1498 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml @@ -15,7 +15,13 @@ rules: resources: - namespaces verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/internal/constellation/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml b/internal/constellation/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml new file mode 100644 index 000000000..0cf4c3f74 --- /dev/null +++ b/internal/constellation/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns-pdb + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns diff --git a/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml index 05f397876..b6fcd3f6b 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-services/charts/join-service/templates/daemonset.yaml @@ -40,6 +40,9 @@ spec: - --cloud-provider=Azure - --key-service-endpoint=key-service.testNamespace:9000 - --attestation-variant=azure-sev-snp + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json volumeMounts: - mountPath: /var/config name: config @@ -47,6 +50,14 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /var/run/state/ssh + name: ssh ports: - containerPort: 9090 name: tcp @@ -54,6 +65,10 @@ spec: securityContext: privileged: true volumes: + - name: gcekey + secret: + secretName: gcekey + optional: true - name: config projected: sources: @@ -64,4 +79,10 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: ssh + hostPath: + path: /var/run/state/ssh updateStrategy: {} diff --git a/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml b/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml deleted file mode 100644 index f189cb6a3..000000000 --- a/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: system:konnectivity-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:konnectivity-server diff --git a/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/daemonset.yaml b/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/daemonset.yaml deleted file mode 100644 index 0f26cfbb9..000000000 --- a/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/daemonset.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: konnectivity-agent - name: konnectivity-agent - namespace: testNamespace -spec: - selector: - matchLabels: - k8s-app: konnectivity-agent - template: - metadata: - labels: - k8s-app: konnectivity-agent - spec: - containers: - - args: - - --logtostderr=true - - --proxy-server-host=127.0.0.1 - - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --proxy-server-port=8132 - - --admin-server-port=8133 - - --health-server-port=8134 - - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token - - --agent-identifiers=host=$(HOST_IP) - - --sync-forever=true - - --keepalive-time=60m - - --sync-interval=5s - - --sync-interval-cap=30s - - --probe-interval=5s - - --v=3 - command: - - /proxy-agent - env: - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - image: konnectivityImage - livenessProbe: - httpGet: - path: /healthz - port: 8134 - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: konnectivity-agent - resources: {} - volumeMounts: - - mountPath: /var/run/secrets/tokens - name: konnectivity-agent-token - readOnly: true - priorityClassName: system-cluster-critical - serviceAccountName: konnectivity-agent - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - name: konnectivity-agent-token - projected: - sources: - - serviceAccountToken: - audience: system:konnectivity-server - path: konnectivity-agent-token - updateStrategy: {} diff --git a/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/serviceaccount.yaml b/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/serviceaccount.yaml deleted file mode 100644 index ad307c56f..000000000 --- a/internal/constellation/helm/testdata/Azure/constellation-services/charts/konnectivity/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: konnectivity-agent - namespace: testNamespace diff --git a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/deployment.yaml b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/deployment.yaml index b8ff3204e..fa41d793b 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/deployment.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/deployment.yaml @@ -37,79 +37,63 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /node-operator - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - - name: CONSTEL_CSP - value: GCP - - name: constellation-uid - value: "42424242424242" - image: constellationOperatorImage - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /etc/kubernetes/pki/etcd - name: etcd-certs - - mountPath: /host/usr/lib/os-release - name: usr-lib-os-release - - mountPath: /etc/os-release - name: etc-os-release - - mountPath: /etc/azure - name: azureconfig - readOnly: true - - mountPath: /etc/gce - name: gceconf - readOnly: true - - mountPath: /etc/constellation-upgrade-agent.sock - name: upgrade-agent-socket - readOnly: true + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8080 + - --leader-elect + command: + - /node-operator + env: + - name: KUBERNETES_CLUSTER_DOMAIN + value: cluster.local + - name: CONSTEL_CSP + value: GCP + - name: constellation-uid + value: "42424242424242" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + image: constellationOperatorImage + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + - mountPath: /host/usr/lib/os-release + name: usr-lib-os-release + - mountPath: /etc/os-release + name: etc-os-release + - mountPath: /etc/azure + name: azureconfig + readOnly: true + - mountPath: /etc/gce + name: gceconf + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /etc/constellation-upgrade-agent.sock + name: upgrade-agent-socket + readOnly: true nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: @@ -117,34 +101,38 @@ spec: serviceAccountName: constellation-operator-controller-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists volumes: - - hostPath: - path: /etc/kubernetes/pki/etcd - type: Directory - name: etcd-certs - - hostPath: - path: /usr/lib/os-release - type: File - name: usr-lib-os-release - - hostPath: - path: /etc/os-release - type: File - name: etc-os-release - - name: azureconfig - secret: - optional: true - secretName: azureconfig - - configMap: + - hostPath: + path: /etc/kubernetes/pki/etcd + type: Directory + name: etcd-certs + - hostPath: + path: /usr/lib/os-release + type: File + name: usr-lib-os-release + - hostPath: + path: /etc/os-release + type: File + name: etc-os-release + - name: azureconfig + secret: + optional: true + secretName: azureconfig + - configMap: + name: gceconf + optional: true name: gceconf - optional: true - name: gceconf - - name: upgrade-agent-socket - hostPath: - path: /run/constellation-upgrade-agent.sock - type: Socket + - name: gcekey + secret: + secretName: gcekey + optional: true + - name: upgrade-agent-socket + hostPath: + path: /run/constellation-upgrade-agent.sock + type: Socket diff --git a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c8..56bf77080 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -16,6 +16,7 @@ rules: verbs: - get - list + - watch - apiGroups: - "" resources: @@ -61,6 +62,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies + - joiningnodes + - nodeversions + - pendingnodes + - scalinggroups verbs: - create - delete @@ -73,38 +78,20 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers + - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes/status + - nodeversions/status + - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -123,84 +110,6 @@ rules: - nodeversion/status verbs: - get -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status - verbs: - - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml index fa9f582a4..e364d1498 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml @@ -15,7 +15,13 @@ rules: resources: - namespaces verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml new file mode 100644 index 000000000..0cf4c3f74 --- /dev/null +++ b/internal/constellation/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/coredns-pdb.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: coredns-pdb + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml index 5855fb988..c0ed7d331 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml @@ -4,4 +4,4 @@ metadata: name: gceconf namespace: testNamespace data: - gce.conf: "[global]\nproject-id = 42424242424242\nuse-metadata-server = true\nnode-tags = constellation-242424242424\nregional = true\n" + gce.conf: "[global]\nproject-id = 42424242424242\nuse-metadata-server = true\nnode-tags = constellation-242424242424\nregional = true\ntoken-url = nil # This forces use of GOOGLE_APPLICATION_CREDENTIALS." diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml index 0ddfa9201..bbe9747ba 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-services/charts/join-service/templates/daemonset.yaml @@ -40,6 +40,9 @@ spec: - --cloud-provider=GCP - --key-service-endpoint=key-service.testNamespace:9000 - --attestation-variant=gcp-sev-es + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json volumeMounts: - mountPath: /var/config name: config @@ -47,6 +50,14 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /var/run/state/ssh + name: ssh ports: - containerPort: 9090 name: tcp @@ -54,6 +65,10 @@ spec: securityContext: privileged: true volumes: + - name: gcekey + secret: + secretName: gcekey + optional: true - name: config projected: sources: @@ -64,4 +79,10 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: ssh + hostPath: + path: /var/run/state/ssh updateStrategy: {} diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml deleted file mode 100644 index f189cb6a3..000000000 --- a/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: system:konnectivity-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:konnectivity-server diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/daemonset.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/daemonset.yaml deleted file mode 100644 index 0f26cfbb9..000000000 --- a/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/daemonset.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: konnectivity-agent - name: konnectivity-agent - namespace: testNamespace -spec: - selector: - matchLabels: - k8s-app: konnectivity-agent - template: - metadata: - labels: - k8s-app: konnectivity-agent - spec: - containers: - - args: - - --logtostderr=true - - --proxy-server-host=127.0.0.1 - - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --proxy-server-port=8132 - - --admin-server-port=8133 - - --health-server-port=8134 - - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token - - --agent-identifiers=host=$(HOST_IP) - - --sync-forever=true - - --keepalive-time=60m - - --sync-interval=5s - - --sync-interval-cap=30s - - --probe-interval=5s - - --v=3 - command: - - /proxy-agent - env: - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - image: konnectivityImage - livenessProbe: - httpGet: - path: /healthz - port: 8134 - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: konnectivity-agent - resources: {} - volumeMounts: - - mountPath: /var/run/secrets/tokens - name: konnectivity-agent-token - readOnly: true - priorityClassName: system-cluster-critical - serviceAccountName: konnectivity-agent - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - name: konnectivity-agent-token - projected: - sources: - - serviceAccountToken: - audience: system:konnectivity-server - path: konnectivity-agent-token - updateStrategy: {} diff --git a/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/serviceaccount.yaml b/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/serviceaccount.yaml deleted file mode 100644 index ad307c56f..000000000 --- a/internal/constellation/helm/testdata/GCP/constellation-services/charts/konnectivity/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: konnectivity-agent - namespace: testNamespace diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/deployment.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/deployment.yaml index b8ff3204e..fa41d793b 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/deployment.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/deployment.yaml @@ -37,79 +37,63 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 - - --leader-elect - command: - - /node-operator - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - - name: CONSTEL_CSP - value: GCP - - name: constellation-uid - value: "42424242424242" - image: constellationOperatorImage - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - name: manager - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /etc/kubernetes/pki/etcd - name: etcd-certs - - mountPath: /host/usr/lib/os-release - name: usr-lib-os-release - - mountPath: /etc/os-release - name: etc-os-release - - mountPath: /etc/azure - name: azureconfig - readOnly: true - - mountPath: /etc/gce - name: gceconf - readOnly: true - - mountPath: /etc/constellation-upgrade-agent.sock - name: upgrade-agent-socket - readOnly: true + - args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=:8080 + - --leader-elect + command: + - /node-operator + env: + - name: KUBERNETES_CLUSTER_DOMAIN + value: cluster.local + - name: CONSTEL_CSP + value: GCP + - name: constellation-uid + value: "42424242424242" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + image: constellationOperatorImage + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /etc/kubernetes/pki/etcd + name: etcd-certs + - mountPath: /host/usr/lib/os-release + name: usr-lib-os-release + - mountPath: /etc/os-release + name: etc-os-release + - mountPath: /etc/azure + name: azureconfig + readOnly: true + - mountPath: /etc/gce + name: gceconf + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /etc/constellation-upgrade-agent.sock + name: upgrade-agent-socket + readOnly: true nodeSelector: node-role.kubernetes.io/control-plane: "" securityContext: @@ -117,34 +101,38 @@ spec: serviceAccountName: constellation-operator-controller-manager terminationGracePeriodSeconds: 10 tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists volumes: - - hostPath: - path: /etc/kubernetes/pki/etcd - type: Directory - name: etcd-certs - - hostPath: - path: /usr/lib/os-release - type: File - name: usr-lib-os-release - - hostPath: - path: /etc/os-release - type: File - name: etc-os-release - - name: azureconfig - secret: - optional: true - secretName: azureconfig - - configMap: + - hostPath: + path: /etc/kubernetes/pki/etcd + type: Directory + name: etcd-certs + - hostPath: + path: /usr/lib/os-release + type: File + name: usr-lib-os-release + - hostPath: + path: /etc/os-release + type: File + name: etc-os-release + - name: azureconfig + secret: + optional: true + secretName: azureconfig + - configMap: + name: gceconf + optional: true name: gceconf - optional: true - name: gceconf - - name: upgrade-agent-socket - hostPath: - path: /run/constellation-upgrade-agent.sock - type: Socket + - name: gcekey + secret: + secretName: gcekey + optional: true + - name: upgrade-agent-socket + hostPath: + path: /run/constellation-upgrade-agent.sock + type: Socket diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c8..56bf77080 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -16,6 +16,7 @@ rules: verbs: - get - list + - watch - apiGroups: - "" resources: @@ -61,6 +62,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies + - joiningnodes + - nodeversions + - pendingnodes + - scalinggroups verbs: - create - delete @@ -73,38 +78,20 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers + - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes/status + - nodeversions/status + - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -123,84 +110,6 @@ rules: - nodeversion/status verbs: - get -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status - verbs: - - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml index fa9f582a4..e364d1498 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml @@ -15,7 +15,13 @@ rules: resources: - namespaces verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml index 0ed907f4d..e680ff691 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/join-service/templates/daemonset.yaml @@ -40,6 +40,9 @@ spec: - --cloud-provider=OpenStack - --key-service-endpoint=key-service.testNamespace:9000 - --attestation-variant=qemu-vtpm + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json volumeMounts: - mountPath: /var/config name: config @@ -47,6 +50,14 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /var/run/state/ssh + name: ssh ports: - containerPort: 9090 name: tcp @@ -54,6 +65,10 @@ spec: securityContext: privileged: true volumes: + - name: gcekey + secret: + secretName: gcekey + optional: true - name: config projected: sources: @@ -64,4 +79,10 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: ssh + hostPath: + path: /var/run/state/ssh updateStrategy: {} diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml deleted file mode 100644 index f189cb6a3..000000000 --- a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: system:konnectivity-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:konnectivity-server diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/daemonset.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/daemonset.yaml deleted file mode 100644 index 0f26cfbb9..000000000 --- a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/daemonset.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: konnectivity-agent - name: konnectivity-agent - namespace: testNamespace -spec: - selector: - matchLabels: - k8s-app: konnectivity-agent - template: - metadata: - labels: - k8s-app: konnectivity-agent - spec: - containers: - - args: - - --logtostderr=true - - --proxy-server-host=127.0.0.1 - - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --proxy-server-port=8132 - - --admin-server-port=8133 - - --health-server-port=8134 - - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token - - --agent-identifiers=host=$(HOST_IP) - - --sync-forever=true - - --keepalive-time=60m - - --sync-interval=5s - - --sync-interval-cap=30s - - --probe-interval=5s - - --v=3 - command: - - /proxy-agent - env: - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - image: konnectivityImage - livenessProbe: - httpGet: - path: /healthz - port: 8134 - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: konnectivity-agent - resources: {} - volumeMounts: - - mountPath: /var/run/secrets/tokens - name: konnectivity-agent-token - readOnly: true - priorityClassName: system-cluster-critical - serviceAccountName: konnectivity-agent - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - name: konnectivity-agent-token - projected: - sources: - - serviceAccountToken: - audience: system:konnectivity-server - path: konnectivity-agent-token - updateStrategy: {} diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/serviceaccount.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/serviceaccount.yaml deleted file mode 100644 index ad307c56f..000000000 --- a/internal/constellation/helm/testdata/OpenStack/constellation-services/charts/konnectivity/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: konnectivity-agent - namespace: testNamespace diff --git a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/deployment.yaml b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/deployment.yaml index 32b81c6c1..edad32c8d 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/deployment.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/deployment.yaml @@ -37,30 +37,9 @@ spec: kubectl.kubernetes.io/default-container: manager spec: containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: cluster.local - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8080 - --leader-elect command: - /node-operator @@ -71,6 +50,8 @@ spec: value: QEMU - name: constellation-uid value: "42424242424242" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json image: constellationOperatorImage livenessProbe: httpGet: @@ -107,6 +88,9 @@ spec: - mountPath: /etc/gce name: gceconf readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true - mountPath: /etc/constellation-upgrade-agent.sock name: upgrade-agent-socket readOnly: true @@ -144,6 +128,10 @@ spec: name: gceconf optional: true name: gceconf + - name: gcekey + secret: + secretName: gcekey + optional: true - name: upgrade-agent-socket hostPath: path: /run/constellation-upgrade-agent.sock diff --git a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c8..56bf77080 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -16,6 +16,7 @@ rules: verbs: - get - list + - watch - apiGroups: - "" resources: @@ -61,6 +62,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies + - joiningnodes + - nodeversions + - pendingnodes + - scalinggroups verbs: - create - delete @@ -73,38 +78,20 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers + - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes/status + - nodeversions/status + - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -123,84 +110,6 @@ rules: - nodeversion/status verbs: - get -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - pendingnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status - verbs: - - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml index fa9f582a4..e364d1498 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/node-maintenance-operator/templates/manager-rbac.yaml @@ -15,7 +15,13 @@ rules: resources: - namespaces verbs: + - create + - delete - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml index 71ad80428..1bd150448 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/join-service/templates/daemonset.yaml @@ -40,6 +40,9 @@ spec: - --cloud-provider=QEMU - --key-service-endpoint=key-service.testNamespace:9000 - --attestation-variant=qemu-vtpm + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json volumeMounts: - mountPath: /var/config name: config @@ -47,6 +50,14 @@ spec: - mountPath: /etc/kubernetes name: kubeadm readOnly: true + - mountPath: /var/kubeadm-config + name: kubeadm-config + readOnly: true + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + - mountPath: /var/run/state/ssh + name: ssh ports: - containerPort: 9090 name: tcp @@ -54,6 +65,10 @@ spec: securityContext: privileged: true volumes: + - name: gcekey + secret: + secretName: gcekey + optional: true - name: config projected: sources: @@ -64,4 +79,10 @@ spec: - name: kubeadm hostPath: path: /etc/kubernetes + - name: kubeadm-config + configMap: + name: kubeadm-config + - name: ssh + hostPath: + path: /var/run/state/ssh updateStrategy: {} diff --git a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml deleted file mode 100644 index f189cb6a3..000000000 --- a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: system:konnectivity-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:konnectivity-server diff --git a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/daemonset.yaml b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/daemonset.yaml deleted file mode 100644 index 0f26cfbb9..000000000 --- a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/daemonset.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: konnectivity-agent - name: konnectivity-agent - namespace: testNamespace -spec: - selector: - matchLabels: - k8s-app: konnectivity-agent - template: - metadata: - labels: - k8s-app: konnectivity-agent - spec: - containers: - - args: - - --logtostderr=true - - --proxy-server-host=127.0.0.1 - - --ca-cert=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - --proxy-server-port=8132 - - --admin-server-port=8133 - - --health-server-port=8134 - - --service-account-token-path=/var/run/secrets/tokens/konnectivity-agent-token - - --agent-identifiers=host=$(HOST_IP) - - --sync-forever=true - - --keepalive-time=60m - - --sync-interval=5s - - --sync-interval-cap=30s - - --probe-interval=5s - - --v=3 - command: - - /proxy-agent - env: - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - image: konnectivityImage - livenessProbe: - httpGet: - path: /healthz - port: 8134 - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: konnectivity-agent - resources: {} - volumeMounts: - - mountPath: /var/run/secrets/tokens - name: konnectivity-agent-token - readOnly: true - priorityClassName: system-cluster-critical - serviceAccountName: konnectivity-agent - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - name: konnectivity-agent-token - projected: - sources: - - serviceAccountToken: - audience: system:konnectivity-server - path: konnectivity-agent-token - updateStrategy: {} diff --git a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/serviceaccount.yaml b/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/serviceaccount.yaml deleted file mode 100644 index ad307c56f..000000000 --- a/internal/constellation/helm/testdata/QEMU/constellation-services/charts/konnectivity/templates/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/cluster-service: "true" - name: konnectivity-agent - namespace: testNamespace diff --git a/internal/constellation/helm/update-aws-load-balancer-chart.sh b/internal/constellation/helm/update-aws-load-balancer-chart.sh index 1c6a8519c..797defaa8 100755 --- a/internal/constellation/helm/update-aws-load-balancer-chart.sh +++ b/internal/constellation/helm/update-aws-load-balancer-chart.sh @@ -8,7 +8,7 @@ set -o errtrace shopt -s inherit_errexit echo "Updating AWS Load Balancer Controller Helm chart..." -branch="v0.0.140" # releases can update the AWS load-balancer-controller chart +branch="v0.0.190" # releases can update the AWS load-balancer-controller chart # Required tools if ! command -v git &> /dev/null; then echo "git could not be found" diff --git a/internal/constellation/helm/update-csi-charts.sh b/internal/constellation/helm/update-csi-charts.sh index 9c2e70531..de3486226 100755 --- a/internal/constellation/helm/update-csi-charts.sh +++ b/internal/constellation/helm/update-csi-charts.sh @@ -27,9 +27,6 @@ fi # $3: path to the Helm chart in the git repo # $4: name of the Helm chart download_chart() { - cleanup() { - rm -r "${repo_tmp_dir}" - } chart_url=$1 branch=$2 chart_dir=$3 @@ -55,31 +52,31 @@ download_chart() { cd "${callDir}" # remove old chart - rm -r "${chart_base_path:?}/${chart_name}" + rm -rf -- "${chart_base_path:?}/${chart_name}" # move new chart mkdir -p "${chart_base_path}/${chart_name}" cp -r "${repo_tmp_dir}/${chart_dir}"/* "${chart_base_path}/${chart_name}" + rm -r -- "${repo_tmp_dir}" + # get new version from Chart.yaml new_version=$(yq '.version' "${chart_base_path}/${chart_name}/Chart.yaml") # update dependency version in parent Chart.yaml yq -i "(.dependencies[] | select( .name== \"${chart_name}\").version) = \"${new_version}\"" "${csi_chart_path}/Chart.yaml" - - return } ## AWS CSI Driver -download_chart "https://github.com/edgelesssys/constellation-aws-ebs-csi-driver" "v1.1.1" "charts/aws-ebs-csi-driver" "aws-csi-driver" +download_chart "https://github.com/edgelesssys/constellation-aws-ebs-csi-driver" "v1.2.0" "charts/aws-ebs-csi-driver" "aws-csi-driver" ## Azure CSI Driver -download_chart "https://github.com/edgelesssys/constellation-azuredisk-csi-driver" "v1.3.0" "charts/edgeless" "azuredisk-csi-driver" +download_chart "https://github.com/edgelesssys/constellation-azuredisk-csi-driver" "v1.4.0" "charts/edgeless" "azuredisk-csi-driver" ## GCP CSI Driver -download_chart "https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver" "v1.3.0" "charts" "gcp-compute-persistent-disk-csi-driver" +download_chart "https://github.com/edgelesssys/constellation-gcp-compute-persistent-disk-csi-driver" "v1.4.0" "charts" "gcp-compute-persistent-disk-csi-driver" ## OpenStack CSI Driver (cinder) -download_chart "https://github.com/edgelesssys/constellation-cloud-provider-openstack" "v1.0.0" "charts/cinder-csi-plugin" "openstack-cinder-csi" +download_chart "https://github.com/edgelesssys/constellation-cloud-provider-openstack" "v1.0.2" "charts/cinder-csi-plugin" "openstack-cinder-csi" echo # final newline diff --git a/internal/constellation/helm/values.go b/internal/constellation/helm/values.go index 264176b73..807f84b0e 100644 --- a/internal/constellation/helm/values.go +++ b/internal/constellation/helm/values.go @@ -1,270 +1,11 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm -import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - -// Values for the Cilium Helm releases for AWS. -var ciliumVals = map[string]map[string]any{ - cloudprovider.AWS.String(): { - "endpointRoutes": map[string]any{ - "enabled": true, - }, - "extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"}, - "encryption": map[string]any{ - "enabled": true, - "type": "wireguard", - "nodeEncryption": true, - "strictMode": map[string]any{ - "enabled": true, - "allowRemoteNodeIdentities": false, - "podCIDRList": []string{"10.244.0.0/16"}, - }, - }, - "l7Proxy": false, - "ipam": map[string]any{ - "operator": map[string]any{ - "clusterPoolIPv4PodCIDRList": []string{ - "10.244.0.0/16", - }, - }, - }, - "image": map[string]any{ - "repository": "ghcr.io/3u13r/cilium", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "digest": "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27", - "useDigest": true, - }, - "operator": map[string]any{ - "image": map[string]any{ - "repository": "ghcr.io/3u13r/operator", - "tag": "v1.15.0-pre.2-edg.1", - "suffix": "", - "genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c", - "useDigest": true, - }, - }, - "bpf": map[string]any{ - "masquerade": true, - }, - "ipMasqAgent": map[string]any{ - "enabled": true, - "config": map[string]any{ - "masqLinkLocal": true, - }, - }, - "kubeProxyReplacement": "strict", - "enableCiliumEndpointSlice": true, - "kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256", - }, - cloudprovider.Azure.String(): { - "endpointRoutes": map[string]any{ - "enabled": true, - }, - "extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"}, - "encryption": map[string]any{ - "enabled": true, - "type": "wireguard", - "nodeEncryption": true, - "strictMode": map[string]any{ - "enabled": true, - "allowRemoteNodeIdentities": false, - "podCIDRList": []string{"10.244.0.0/16"}, - }, - }, - "l7Proxy": false, - "ipam": map[string]any{ - "operator": map[string]any{ - "clusterPoolIPv4PodCIDRList": []string{ - "10.244.0.0/16", - }, - }, - }, - "image": map[string]any{ - "repository": "ghcr.io/3u13r/cilium", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "digest": "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27", - "useDigest": true, - }, - "operator": map[string]any{ - "image": map[string]any{ - "repository": "ghcr.io/3u13r/operator", - "tag": "v1.15.0-pre.2-edg.1", - "suffix": "", - "genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c", - "useDigest": true, - }, - }, - "bpf": map[string]any{ - "masquerade": true, - }, - "ipMasqAgent": map[string]any{ - "enabled": true, - "config": map[string]any{ - "masqLinkLocal": true, - }, - }, - "kubeProxyReplacement": "strict", - "enableCiliumEndpointSlice": true, - "kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256", - }, - cloudprovider.GCP.String(): { - "endpointRoutes": map[string]any{ - "enabled": true, - }, - "extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"}, - "tunnel": "disabled", - "encryption": map[string]any{ - "enabled": true, - "type": "wireguard", - "nodeEncryption": true, - "strictMode": map[string]any{ - "enabled": true, - "allowRemoteNodeIdentities": false, - }, - }, - "image": map[string]any{ - "repository": "ghcr.io/3u13r/cilium", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "digest": "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27", - "useDigest": true, - }, - "operator": map[string]any{ - "image": map[string]any{ - "repository": "ghcr.io/3u13r/operator", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c", - "useDigest": true, - }, - }, - "l7Proxy": false, - "ipam": map[string]any{ - "mode": "kubernetes", - }, - "bpf": map[string]any{ - "masquerade": true, - }, - "ipMasqAgent": map[string]any{ - "enabled": true, - "config": map[string]any{ - "masqLinkLocal": true, - }, - }, - "kubeProxyReplacement": "strict", - "enableCiliumEndpointSlice": true, - "kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256", - }, - cloudprovider.OpenStack.String(): { - "endpointRoutes": map[string]any{ - "enabled": true, - }, - "extraArgs": []string{"--node-encryption-opt-out-labels=invalid.label"}, - "encryption": map[string]any{ - "enabled": true, - "type": "wireguard", - "nodeEncryption": true, - "strictMode": map[string]any{ - "enabled": true, - "podCIDRList": []string{"10.244.0.0/16"}, - }, - }, - "l7Proxy": false, - "ipam": map[string]any{ - "operator": map[string]any{ - "clusterPoolIPv4PodCIDRList": []string{ - "10.244.0.0/16", - }, - }, - }, - "image": map[string]any{ - "repository": "ghcr.io/3u13r/cilium", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "digest": "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27", - "useDigest": true, - }, - "operator": map[string]any{ - "image": map[string]any{ - "repository": "ghcr.io/3u13r/operator", - "tag": "v1.15.0-pre.2-edg.1", - "suffix": "", - "genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c", - "useDigest": true, - }, - }, - "bpf": map[string]any{ - "masquerade": true, - }, - "ipMasqAgent": map[string]any{ - "enabled": true, - "config": map[string]any{ - "masqLinkLocal": true, - }, - }, - "kubeProxyReplacement": "strict", - "enableCiliumEndpointSlice": true, - "kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256", - }, - cloudprovider.QEMU.String(): { - "endpointRoutes": map[string]any{ - "enabled": true, - }, - "encryption": map[string]any{ - "enabled": true, - "type": "wireguard", - "nodeEncryption": true, - "strictMode": map[string]any{ - "enabled": true, - "podCIDRList": []string{"10.244.0.0/16"}, - }, - }, - "image": map[string]any{ - "repository": "ghcr.io/3u13r/cilium", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "digest": "sha256:eebf631fd0f27e1f28f1fdeb2e049f2c83b887381466245c4b3e26440daefa27", - "useDigest": true, - }, - "operator": map[string]any{ - "image": map[string]any{ - "repository": "ghcr.io/3u13r/operator", - "suffix": "", - "tag": "v1.15.0-pre.2-edg.1", - "genericDigest": "sha256:bfaeac2e05e8c38f439b0fbc36558fd8d11602997f2641423e8d86bd7ac6a88c", - "useDigest": true, - }, - }, - "ipam": map[string]any{ - "operator": map[string]any{ - "clusterPoolIPv4PodCIDRList": []string{ - "10.244.0.0/16", - }, - }, - }, - "bpf": map[string]any{ - "masquerade": true, - }, - "ipMasqAgent": map[string]any{ - "enabled": true, - "config": map[string]any{ - "masqLinkLocal": true, - }, - }, - "kubeProxyReplacement": "strict", - "enableCiliumEndpointSlice": true, - "kubeProxyReplacementHealthzBindAddr": "0.0.0.0:10256", - "l7Proxy": false, - }, -} - var controlPlaneNodeSelector = map[string]any{"node-role.kubernetes.io/control-plane": ""} var controlPlaneTolerations = []map[string]any{ diff --git a/internal/constellation/helm/versionlister.go b/internal/constellation/helm/versionlister.go index 526cfebe3..c5faf1aea 100644 --- a/internal/constellation/helm/versionlister.go +++ b/internal/constellation/helm/versionlister.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package helm diff --git a/internal/constellation/kubecmd/BUILD.bazel b/internal/constellation/kubecmd/BUILD.bazel index 4eef4832d..aca26d0bb 100644 --- a/internal/constellation/kubecmd/BUILD.bazel +++ b/internal/constellation/kubecmd/BUILD.bazel @@ -30,8 +30,11 @@ go_library( "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/runtime/serializer/json", "@io_k8s_client_go//util/retry", - "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/scheme", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta4", "@io_k8s_sigs_yaml//:yaml", ], ) @@ -66,7 +69,6 @@ go_test( "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", - "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_sigs_yaml//:yaml", ], ) diff --git a/internal/constellation/kubecmd/backup.go b/internal/constellation/kubecmd/backup.go index 2a396da8b..93a9cada7 100644 --- a/internal/constellation/kubecmd/backup.go +++ b/internal/constellation/kubecmd/backup.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubecmd @@ -26,7 +26,7 @@ type crdLister interface { // BackupCRDs backs up all CRDs to the upgrade workspace. func (k *KubeCmd) BackupCRDs(ctx context.Context, fileHandler file.Handler, upgradeDir string) ([]apiextensionsv1.CustomResourceDefinition, error) { - k.log.Debugf("Starting CRD backup") + k.log.Debug("Starting CRD backup") crds, err := k.kubectl.ListCRDs(ctx) if err != nil { return nil, fmt.Errorf("getting CRDs: %w", err) @@ -39,7 +39,7 @@ func (k *KubeCmd) BackupCRDs(ctx context.Context, fileHandler file.Handler, upgr for i := range crds { path := filepath.Join(crdBackupFolder, crds[i].Name+".yaml") - k.log.Debugf("Creating CRD backup: %s", path) + k.log.Debug("Creating CRD backup", "path", path) // We have to manually set kind/apiversion because of a long-standing limitation of the API: // https://github.com/kubernetes/kubernetes/issues/3030#issuecomment-67543738 @@ -56,15 +56,15 @@ func (k *KubeCmd) BackupCRDs(ctx context.Context, fileHandler file.Handler, upgr return nil, err } } - k.log.Debugf("CRD backup complete") + k.log.Debug("CRD backup complete") return crds, nil } // BackupCRs backs up all CRs to the upgrade workspace. func (k *KubeCmd) BackupCRs(ctx context.Context, fileHandler file.Handler, crds []apiextensionsv1.CustomResourceDefinition, upgradeDir string) error { - k.log.Debugf("Starting CR backup") + k.log.Debug("Starting CR backup") for _, crd := range crds { - k.log.Debugf("Creating backup for resource type: %s", crd.Name) + k.log.Debug("Creating backup", "crdName", crd.Name) // Iterate over all versions of the CRD // TODO(daniel-weisse): Consider iterating over crd.Status.StoredVersions instead @@ -72,7 +72,7 @@ func (k *KubeCmd) BackupCRs(ctx context.Context, fileHandler file.Handler, crds // a version that is not installed in the cluster. // With the StoredVersions field, we could only iterate over the installed versions. for _, version := range crd.Spec.Versions { - k.log.Debugf("Creating backup of CRs for %q at version %q", crd.Name, version.Name) + k.log.Debug("Starting CustomResource backup", "crdName", crd.Name, "version", version.Name) gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: version.Name, Resource: crd.Spec.Names.Plural} crs, err := k.kubectl.ListCRs(ctx, gvr) @@ -80,7 +80,7 @@ func (k *KubeCmd) BackupCRs(ctx context.Context, fileHandler file.Handler, crds if !k8serrors.IsNotFound(err) { return fmt.Errorf("retrieving CR %s: %w", crd.Name, err) } - k.log.Debugf("No CRs found for %q at version %q, skipping...", crd.Name, version.Name) + k.log.Debug("No CustomResources found. Skipping...", "crdName", crd.Name, "version", version.Name) continue } @@ -101,9 +101,9 @@ func (k *KubeCmd) BackupCRs(ctx context.Context, fileHandler file.Handler, crds } } - k.log.Debugf("Backup for resource type %q complete", crd.Name) + k.log.Debug("CustomResource backup complete", "crdName", crd.Name) } - k.log.Debugf("CR backup complete") + k.log.Debug("All CustomResource backups completed") return nil } diff --git a/internal/constellation/kubecmd/backup_test.go b/internal/constellation/kubecmd/backup_test.go index 21a3dc65b..ac6e42d54 100644 --- a/internal/constellation/kubecmd/backup_test.go +++ b/internal/constellation/kubecmd/backup_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubecmd @@ -57,7 +57,7 @@ func TestBackupCRDs(t *testing.T) { log: stubLog{}, } - _, err = client.BackupCRDs(context.Background(), file.NewHandler(memFs), tc.upgradeID) + _, err = client.BackupCRDs(t.Context(), file.NewHandler(memFs), tc.upgradeID) if tc.wantError { assert.Error(err) return @@ -146,7 +146,7 @@ func TestBackupCRs(t *testing.T) { log: stubLog{}, } - err := client.BackupCRs(context.Background(), file.NewHandler(memFs), []apiextensionsv1.CustomResourceDefinition{tc.crd}, tc.upgradeID) + err := client.BackupCRs(t.Context(), file.NewHandler(memFs), []apiextensionsv1.CustomResourceDefinition{tc.crd}, tc.upgradeID) if tc.wantError { assert.Error(err) return @@ -166,8 +166,7 @@ func TestBackupCRs(t *testing.T) { type stubLog struct{} -func (s stubLog) Debugf(_ string, _ ...any) {} -func (s stubLog) Sync() {} +func (s stubLog) Debug(_ string, _ ...any) {} func (c stubKubectl) ListCRDs(_ context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) { if c.getCRDsError != nil { diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index ea49bef2f..7fb911e44 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "slices" "sort" "strings" "time" @@ -36,20 +35,18 @@ import ( "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions/components" - updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/util/retry" - kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" - "sigs.k8s.io/yaml" -) - -const ( - maxRetryAttempts = 20 + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + kubeadmv1beta4 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4" ) // ErrInProgress signals that an upgrade is in progress inside the cluster. @@ -70,6 +67,7 @@ func (e *applyError) Error() string { type KubeCmd struct { kubectl kubectlInterface retryInterval time.Duration + maxAttempts int log debugLog } @@ -83,6 +81,7 @@ func New(kubeConfig []byte, log debugLog) (*KubeCmd, error) { return &KubeCmd{ kubectl: client, retryInterval: time.Second * 5, + maxAttempts: 20, log: log, }, nil } @@ -94,7 +93,7 @@ func (k *KubeCmd) UpgradeNodeImage(ctx context.Context, imageVersion semver.Semv return err } - k.log.Debugf("Checking if image upgrade is valid") + k.log.Debug("Checking if image upgrade is valid") var upgradeErr *compatibility.InvalidUpgradeError err = k.isValidImageUpgrade(nodeVersion, imageVersion.String(), force) switch { @@ -104,12 +103,7 @@ func (k *KubeCmd) UpgradeNodeImage(ctx context.Context, imageVersion semver.Semv return fmt.Errorf("updating image version: %w", err) } - // TODO(3u13r): remove `reconcileKubeadmConfigMap` after v2.14.0 has been released. - if err := k.reconcileKubeadmConfigMap(ctx); err != nil { - return fmt.Errorf("reconciling kubeadm config: %w", err) - } - - k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.String()) + k.log.Debug("Updating local copy of nodeVersion image version", "oldVersion", nodeVersion.Spec.ImageVersion, "newVersion", imageVersion.String()) nodeVersion.Spec.ImageReference = imageReference nodeVersion.Spec.ImageVersion = imageVersion.String() @@ -127,41 +121,43 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio return err } - var upgradeErr *compatibility.InvalidUpgradeError // We have to allow users to specify outdated k8s patch versions. // Therefore, this code has to skip k8s updates if a user configures an outdated (i.e. invalid) k8s version. - var components *corev1.ConfigMap - _, err = versions.NewValidK8sVersion(string(kubernetesVersion), true) - if err != nil { - err = compatibility.NewInvalidUpgradeError( + if _, err := versions.NewValidK8sVersion(string(kubernetesVersion), true); err != nil { + return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError( nodeVersion.Spec.KubernetesClusterVersion, string(kubernetesVersion), - fmt.Errorf("unsupported Kubernetes version, supported versions are %s", strings.Join(versions.SupportedK8sVersions(), ", ")), + fmt.Errorf("unsupported Kubernetes version, supported versions are %s", strings.Join(versions.SupportedK8sVersions(), ", "))), ) - } else { - versionConfig, ok := versions.VersionConfigs[kubernetesVersion] - if !ok { - err = compatibility.NewInvalidUpgradeError( - nodeVersion.Spec.KubernetesClusterVersion, - string(kubernetesVersion), - fmt.Errorf("no version config matching K8s %s", kubernetesVersion), - ) - } else { - components, err = k.prepareUpdateK8s(&nodeVersion, versionConfig.ClusterVersion, - versionConfig.KubernetesComponents, force) - } } - switch { - case err == nil: - err := k.applyComponentsCM(ctx, components) - if err != nil { - return fmt.Errorf("applying k8s components ConfigMap: %w", err) + // TODO(burgerdev): remove after releasing v2.19 + // Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to + // connect to the local API server. + if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) { + if cc.FeatureGates == nil { + cc.FeatureGates = map[string]bool{} } - case errors.As(err, &upgradeErr): - return fmt.Errorf("skipping Kubernetes upgrade: %w", err) - default: - return fmt.Errorf("updating Kubernetes version: %w", err) + cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true + }); err != nil { + return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err) + } + + versionConfig, ok := versions.VersionConfigs[kubernetesVersion] + if !ok { + return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError( + nodeVersion.Spec.KubernetesClusterVersion, + string(kubernetesVersion), + fmt.Errorf("no version config matching K8s %s", kubernetesVersion), + )) + } + components, err := k.prepareUpdateK8s(&nodeVersion, versionConfig.ClusterVersion, versionConfig.KubernetesComponents, force) + if err != nil { + return err + } + + if err := k.applyComponentsCM(ctx, components); err != nil { + return fmt.Errorf("applying k8s components ConfigMap: %w", err) } updatedNodeVersion, err := k.applyNodeVersion(ctx, nodeVersion) @@ -173,8 +169,13 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio // ClusterStatus returns a map from node name to NodeStatus. func (k *KubeCmd) ClusterStatus(ctx context.Context) (map[string]NodeStatus, error) { - nodes, err := k.kubectl.GetNodes(ctx) - if err != nil { + var nodes []corev1.Node + + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + nodes, err = k.kubectl.GetNodes(ctx) + return err + }); err != nil { return nil, fmt.Errorf("getting nodes: %w", err) } @@ -189,7 +190,7 @@ func (k *KubeCmd) ClusterStatus(ctx context.Context) (map[string]NodeStatus, err // GetClusterAttestationConfig fetches the join-config configmap from the cluster, // and returns the attestation config. func (k *KubeCmd) GetClusterAttestationConfig(ctx context.Context, variant variant.Variant) (config.AttestationCfg, error) { - existingConf, err := retryGetJoinConfig(ctx, k.kubectl, k.retryInterval, k.log) + existingConf, err := k.retryGetJoinConfig(ctx) if err != nil { return nil, fmt.Errorf("retrieving current attestation config: %w", err) } @@ -214,30 +215,30 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At return fmt.Errorf("marshaling attestation config: %w", err) } - joinConfig, err := retryGetJoinConfig(ctx, k.kubectl, k.retryInterval, k.log) + joinConfig, err := k.retryGetJoinConfig(ctx) if err != nil { if !k8serrors.IsNotFound(err) { return fmt.Errorf("getting %s ConfigMap: %w", constants.JoinConfigMap, err) } - k.log.Debugf("ConfigMap %q does not exist in namespace %q, creating it now", constants.JoinConfigMap, constants.ConstellationNamespace) - if err := retryAction(ctx, k.retryInterval, maxRetryAttempts, func(ctx context.Context) error { + k.log.Debug("ConfigMap does not exist, creating it now", "name", constants.JoinConfigMap, "namespace", constants.ConstellationNamespace) + if err := k.retryAction(ctx, func(ctx context.Context) error { return k.kubectl.CreateConfigMap(ctx, joinConfigMap(newConfigJSON, measurementSalt)) - }, k.log); err != nil { + }); err != nil { return fmt.Errorf("creating join-config ConfigMap: %w", err) } - k.log.Debugf("Created %q ConfigMap in namespace %q", constants.JoinConfigMap, constants.ConstellationNamespace) + k.log.Debug("Created ConfigMap", "name", constants.JoinConfigMap, "namespace", constants.ConstellationNamespace) return nil } // create backup of previous config joinConfig.Data[constants.AttestationConfigFilename+"_backup"] = joinConfig.Data[constants.AttestationConfigFilename] joinConfig.Data[constants.AttestationConfigFilename] = string(newConfigJSON) - k.log.Debugf("Triggering attestation config update now") - if err := retryAction(ctx, k.retryInterval, maxRetryAttempts, func(ctx context.Context) error { + k.log.Debug("Triggering attestation config update now") + if err := k.retryAction(ctx, func(ctx context.Context) error { _, err = k.kubectl.UpdateConfigMap(ctx, joinConfig) return err - }, k.log); err != nil { + }); err != nil { return fmt.Errorf("setting new attestation config: %w", err) } @@ -247,48 +248,35 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) - if err != nil { - return fmt.Errorf("getting ClusterConfig: %w", err) - } - - existingSANs := make(map[string]struct{}) - for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { - existingSANs[existingSAN] = struct{}{} - } - - var missingSANs []string - for _, san := range alternativeNames { - if san == "" { - continue // skip empty SANs + if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) { + existingSANs := make(map[string]struct{}) + for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { + existingSANs[existingSAN] = struct{}{} } - if _, ok := existingSANs[san]; !ok { - missingSANs = append(missingSANs, san) - existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice + + var missingSANs []string + for _, san := range alternativeNames { + if san == "" { + continue // skip empty SANs + } + if _, ok := existingSANs[san]; !ok { + missingSANs = append(missingSANs, san) + existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice + } } + + if len(missingSANs) == 0 { + k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") + } + k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) + + clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) + sort.Strings(clusterConfiguration.APIServer.CertSANs) + }); err != nil { + return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err) } - if len(missingSANs) == 0 { - k.log.Debugf("No new SANs to add to the cluster's apiserver SAN field") - return nil - } - k.log.Debugf("Extending the cluster's apiserver SAN field with the following SANs: %s\n", strings.Join(missingSANs, ", ")) - - clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) - sort.Strings(clusterConfiguration.APIServer.CertSANs) - - newConfigYAML, err := yaml.Marshal(clusterConfiguration) - if err != nil { - return fmt.Errorf("marshaling ClusterConfiguration: %w", err) - } - - kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) - k.log.Debugf("Triggering kubeadm config update now") - if _, err = k.kubectl.UpdateConfigMap(ctx, kubeadmConfig); err != nil { - return fmt.Errorf("setting new kubeadm config: %w", err) - } - - k.log.Debugf("Successfully extended the cluster's apiserver SAN field") + k.log.Debug("Successfully extended the cluster's apiserver SAN field") return nil } @@ -305,14 +293,19 @@ func (k *KubeCmd) GetConstellationVersion(ctx context.Context) (NodeVersion, err // getConstellationVersion returns the NodeVersion object of a Constellation cluster. func (k *KubeCmd) getConstellationVersion(ctx context.Context) (updatev1alpha1.NodeVersion, error) { - raw, err := k.kubectl.GetCR(ctx, schema.GroupVersionResource{ - Group: "update.edgeless.systems", - Version: "v1alpha1", - Resource: "nodeversions", - }, "constellation-version") - if err != nil { + var raw *unstructured.Unstructured + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + raw, err = k.kubectl.GetCR(ctx, schema.GroupVersionResource{ + Group: "update.edgeless.systems", + Version: "v1alpha1", + Resource: "nodeversions", + }, constants.NodeVersionResourceName) + return err + }); err != nil { return updatev1alpha1.NodeVersion{}, err } + var nodeVersion updatev1alpha1.NodeVersion if err := runtime.DefaultUnstructuredConverter.FromUnstructured(raw.UnstructuredContent(), &nodeVersion); err != nil { return updatev1alpha1.NodeVersion{}, fmt.Errorf("converting unstructured to NodeVersion: %w", err) @@ -321,106 +314,59 @@ func (k *KubeCmd) getConstellationVersion(ctx context.Context) (updatev1alpha1.N return nodeVersion, nil } -// getClusterConfiguration fetches the kubeadm-config configmap from the cluster, extracts the config -// and returns both the full configmap and the ClusterConfiguration. -func (k *KubeCmd) getClusterConfiguration(ctx context.Context) (kubeadmv1beta3.ClusterConfiguration, *corev1.ConfigMap, error) { - existingConf, err := k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - if err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - clusterConf, ok := existingConf.Data[constants.ClusterConfigurationKey] - if !ok { - return kubeadmv1beta3.ClusterConfiguration{}, nil, errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var existingClusterConfig kubeadmv1beta3.ClusterConfiguration - if err := yaml.Unmarshal([]byte(clusterConf), &existingClusterConfig); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("unmarshaling ClusterConfiguration: %w", err) - } - - return existingClusterConfig, existingConf, nil -} - // applyComponentsCM applies the k8s components ConfigMap to the cluster. func (k *KubeCmd) applyComponentsCM(ctx context.Context, components *corev1.ConfigMap) error { - // If the map already exists we can use that map and assume it has the same content as 'configMap'. - if err := k.kubectl.CreateConfigMap(ctx, components); err != nil && !k8serrors.IsAlreadyExists(err) { - return fmt.Errorf("creating k8s-components ConfigMap: %w. %T", err, err) + if err := k.retryAction(ctx, func(ctx context.Context) error { + // If the components ConfigMap already exists we assume it is up to date, + // since its name is derived from a hash of its contents. + err := k.kubectl.CreateConfigMap(ctx, components) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return err + } + return nil + }); err != nil { + return fmt.Errorf("creating k8s-components ConfigMap: %w", err) } return nil } func (k *KubeCmd) applyNodeVersion(ctx context.Context, nodeVersion updatev1alpha1.NodeVersion) (updatev1alpha1.NodeVersion, error) { - k.log.Debugf("Triggering NodeVersion upgrade now") + k.log.Debug("Triggering NodeVersion upgrade now") var updatedNodeVersion updatev1alpha1.NodeVersion - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - newNode, err := k.getConstellationVersion(ctx) - if err != nil { - return fmt.Errorf("retrieving current NodeVersion: %w", err) - } - updateNodeVersions(nodeVersion, &newNode) + // Retry the entire "retry-on-conflict" block to retry if the block fails, e.g. due to etcd timeouts. + err := k.retryAction(ctx, func(ctx context.Context) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + newNode, err := k.getConstellationVersion(ctx) + if err != nil { + return fmt.Errorf("retrieving current NodeVersion: %w", err) + } - raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&newNode) - if err != nil { - return fmt.Errorf("converting nodeVersion to unstructured: %w", err) - } - updated, err := k.kubectl.UpdateCR(ctx, schema.GroupVersionResource{ - Group: "update.edgeless.systems", - Version: "v1alpha1", - Resource: "nodeversions", - }, &unstructured.Unstructured{Object: raw}) - if err != nil { - return err - } + updateNodeVersions(nodeVersion, &newNode) - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(updated.UnstructuredContent(), &updatedNodeVersion); err != nil { - return fmt.Errorf("converting unstructured to NodeVersion: %w", err) - } - return nil + raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&newNode) + if err != nil { + return fmt.Errorf("converting nodeVersion to unstructured: %w", err) + } + updated, err := k.kubectl.UpdateCR(ctx, schema.GroupVersionResource{ + Group: "update.edgeless.systems", + Version: "v1alpha1", + Resource: "nodeversions", + }, &unstructured.Unstructured{Object: raw}) + if err != nil { + return err + } + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(updated.UnstructuredContent(), &updatedNodeVersion); err != nil { + return fmt.Errorf("converting unstructured to NodeVersion: %w", err) + } + return nil + }) }) return updatedNodeVersion, err } -func (k *KubeCmd) reconcileKubeadmConfigMap(ctx context.Context) error { - clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) - if err != nil { - return fmt.Errorf("getting ClusterConfig: %w", err) - } - - for i, v := range clusterConfiguration.APIServer.ExtraVolumes { - if v.Name == "konnectivity-uds" { - clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1) - } - } - for i, v := range clusterConfiguration.APIServer.ExtraVolumes { - if v.Name == "egress-config" { - clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1) - } - } - delete(clusterConfiguration.APIServer.ExtraArgs, "egress-selector-config-file") - - newConfigYAML, err := yaml.Marshal(clusterConfiguration) - if err != nil { - return fmt.Errorf("marshaling ClusterConfiguration: %w", err) - } - - if kubeadmConfig.Data[constants.ClusterConfigurationKey] == string(newConfigYAML) { - k.log.Debugf("No changes to kubeadm config required") - return nil - } - - kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) - k.log.Debugf("Triggering kubeadm config update now") - if _, err = k.kubectl.UpdateConfigMap(ctx, kubeadmConfig); err != nil { - return fmt.Errorf("setting new kubeadm config: %w", err) - } - - k.log.Debugf("Successfully reconciled the cluster's kubeadm config") - return nil -} - // isValidImageUpdate checks if the new image version is a valid upgrade, and there is no upgrade already running. func (k *KubeCmd) isValidImageUpgrade(nodeVersion updatev1alpha1.NodeVersion, newImageVersion string, force bool) error { if !force { @@ -449,17 +395,97 @@ func (k *KubeCmd) prepareUpdateK8s(nodeVersion *updatev1alpha1.NodeVersion, newC } if !force { if err := compatibility.IsValidUpgrade(nodeVersion.Spec.KubernetesClusterVersion, newClusterVersion); err != nil { - return nil, err + return nil, fmt.Errorf("skipping Kubernetes upgrade: %w", err) } } - k.log.Debugf("Updating local copy of nodeVersion Kubernetes version from %s to %s", nodeVersion.Spec.KubernetesClusterVersion, newClusterVersion) + k.log.Debug("Updating local copy of nodeVersion Kubernetes version", "oldVersion", nodeVersion.Spec.KubernetesClusterVersion, "newVersion", newClusterVersion) nodeVersion.Spec.KubernetesComponentsReference = configMap.ObjectMeta.Name nodeVersion.Spec.KubernetesClusterVersion = newClusterVersion return &configMap, nil } +func (k *KubeCmd) retryGetJoinConfig(ctx context.Context) (*corev1.ConfigMap, error) { + var ctr int + retrieable := func(err error) bool { + if k8serrors.IsNotFound(err) { + return false + } + ctr++ + k.log.Debug("Getting join-config ConfigMap failed", "attempt", ctr, "maxAttempts", k.maxAttempts, "error", err) + return ctr < k.maxAttempts + } + + var joinConfig *corev1.ConfigMap + var err error + doer := &kubeDoer{ + action: func(ctx context.Context) error { + joinConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap) + return err + }, + } + retrier := conretry.NewIntervalRetrier(doer, k.retryInterval, retrieable) + + err = retrier.Do(ctx) + return joinConfig, err +} + +func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Context) error) error { + ctr := 0 + retrier := conretry.NewIntervalRetrier(&kubeDoer{action: action}, k.retryInterval, func(err error) bool { + ctr++ + k.log.Debug("Action failed", "attempt", ctr, "maxAttempts", k.maxAttempts, "error", err) + return ctr < k.maxAttempts + }) + return retrier.Do(ctx) +} + +// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry, +// runs doPatch on it and uploads the result. +func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error { + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) + } + + doPatch(&clusterConfiguration) + + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debug("Triggering kubeadm config update now") + if err = k.retryAction(ctx, func(ctx context.Context) error { + _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) + return err + }); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + k.log.Debug("Successfully patched the cluster's kubeadm-config") + return nil +} + func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error { var err error switch { @@ -498,41 +524,6 @@ func (k *kubeDoer) Do(ctx context.Context) error { return k.action(ctx) } -func retryGetJoinConfig(ctx context.Context, kubectl kubectlInterface, retryInterval time.Duration, log debugLog) (*corev1.ConfigMap, error) { - var retries int - retrieable := func(err error) bool { - if k8serrors.IsNotFound(err) { - return false - } - retries++ - log.Debugf("Getting join-config ConfigMap failed (attempt %d/%d): %s", retries, maxRetryAttempts, err) - return retries < maxRetryAttempts - } - - var joinConfig *corev1.ConfigMap - var err error - doer := &kubeDoer{ - action: func(ctx context.Context) error { - joinConfig, err = kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.JoinConfigMap) - return err - }, - } - retrier := conretry.NewIntervalRetrier(doer, retryInterval, retrieable) - - err = retrier.Do(ctx) - return joinConfig, err -} - -func retryAction(ctx context.Context, retryInterval time.Duration, maxRetries int, action func(ctx context.Context) error, log debugLog) error { - ctr := 0 - retrier := conretry.NewIntervalRetrier(&kubeDoer{action: action}, retryInterval, func(err error) bool { - ctr++ - log.Debugf("Action failed (attempt %d/%d): %s", ctr, maxRetries, err) - return ctr < maxRetries - }) - return retrier.Do(ctx) -} - // kubectlInterface provides access to the Kubernetes API. type kubectlInterface interface { GetNodes(ctx context.Context) ([]corev1.Node, error) @@ -546,5 +537,5 @@ type kubectlInterface interface { } type debugLog interface { - Debugf(format string, args ...any) + Debug(msg string, args ...any) } diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index cc334bfed..3e861afb7 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubecmd @@ -11,7 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "strings" "testing" "time" @@ -23,7 +22,7 @@ import ( "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" "github.com/edgelesssys/constellation/v2/internal/versions/components" - updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -34,80 +33,21 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) func TestUpgradeNodeImage(t *testing.T) { - clusterConf := kubeadmv1beta3.ClusterConfiguration{ - APIServer: kubeadmv1beta3.APIServer{ - ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ - ExtraArgs: map[string]string{}, - ExtraVolumes: []kubeadmv1beta3.HostPathMount{}, - }, - }, - } - - clusterConfBytes, err := json.Marshal(clusterConf) - require.NoError(t, err) - validKubeadmConfig := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.KubeadmConfigMap, - }, - Data: map[string]string{ - constants.ClusterConfigurationKey: string(clusterConfBytes), - }, - } - - clusterConfWithKonnectivity := kubeadmv1beta3.ClusterConfiguration{ - APIServer: kubeadmv1beta3.APIServer{ - ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "egress-selector-config-file": "/etc/kubernetes/egress-selector-config-file.yaml", - }, - ExtraVolumes: []kubeadmv1beta3.HostPathMount{ - { - Name: "egress-config", - HostPath: "/etc/kubernetes/egress-selector-config-file.yaml", - }, - { - Name: "konnectivity-uds", - HostPath: "/some/path/to/konnectivity-uds", - }, - }, - }, - }, - } - - clusterConfBytesWithKonnectivity, err := json.Marshal(clusterConfWithKonnectivity) - require.NoError(t, err) - validKubeadmConfigWithKonnectivity := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.KubeadmConfigMap, - }, - Data: map[string]string{ - constants.ClusterConfigurationKey: string(clusterConfBytesWithKonnectivity), - }, - } - testCases := map[string]struct { conditions []metav1.Condition currentImageVersion semver.Semver newImageVersion semver.Semver badImageVersion string force bool - customKubeadmConfig *corev1.ConfigMap getCRErr error wantErr bool wantUpdate bool assertCorrectError func(t *testing.T, err error) bool customClientFn func(nodeVersion updatev1alpha1.NodeVersion) unstructuredInterface }{ - "success with konnectivity migration": { - currentImageVersion: semver.NewFromInt(1, 2, 2, ""), - newImageVersion: semver.NewFromInt(1, 2, 3, ""), - customKubeadmConfig: validKubeadmConfigWithKonnectivity, - wantUpdate: true, - }, "success": { currentImageVersion: semver.NewFromInt(1, 2, 2, ""), newImageVersion: semver.NewFromInt(1, 2, 3, ""), @@ -226,22 +166,21 @@ func TestUpgradeNodeImage(t *testing.T) { kubectl := &stubKubectl{ unstructuredInterface: unstructuredClient, configMaps: map[string]*corev1.ConfigMap{ - constants.KubeadmConfigMap: validKubeadmConfig, + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), }, } if tc.customClientFn != nil { kubectl.unstructuredInterface = tc.customClientFn(nodeVersion) } - if tc.customKubeadmConfig != nil { - kubectl.configMaps[constants.KubeadmConfigMap] = tc.customKubeadmConfig - } upgrader := KubeCmd{ - kubectl: kubectl, - log: logger.NewTest(t), + kubectl: kubectl, + retryInterval: time.Millisecond, + maxAttempts: 5, + log: logger.NewTest(t), } - err = upgrader.UpgradeNodeImage(context.Background(), tc.newImageVersion, fmt.Sprintf("/path/to/image:%s", tc.newImageVersion.String()), tc.force) + err = upgrader.UpgradeNodeImage(t.Context(), tc.newImageVersion, fmt.Sprintf("/path/to/image:%s", tc.newImageVersion.String()), tc.force) // Check upgrades first because if we checked err first, UpgradeImage may error due to other reasons and still trigger an upgrade. if tc.wantUpdate { assert.NotNil(unstructuredClient.updatedObject) @@ -255,12 +194,6 @@ func TestUpgradeNodeImage(t *testing.T) { return } assert.NoError(err) - // If the ConfigMap only exists in the updatedConfigMaps map, the Konnectivity values should have been removed - if strings.Contains(kubectl.configMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") { - assert.NotContains(kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") - assert.NotContains(kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config") - assert.NotContains(kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file") - } }) } } @@ -348,17 +281,22 @@ func TestUpgradeKubernetesVersion(t *testing.T) { } kubectl := &stubKubectl{ unstructuredInterface: unstructuredClient, + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}}, + }, } if tc.customClientFn != nil { kubectl.unstructuredInterface = tc.customClientFn(nodeVersion) } upgrader := KubeCmd{ - kubectl: kubectl, - log: logger.NewTest(t), + kubectl: kubectl, + retryInterval: time.Millisecond, + maxAttempts: 5, + log: logger.NewTest(t), } - err = upgrader.UpgradeKubernetesVersion(context.Background(), tc.newKubernetesVersion, tc.force) + err = upgrader.UpgradeKubernetesVersion(t.Context(), tc.newKubernetesVersion, tc.force) // Check upgrades first because if we checked err first, UpgradeImage may error due to other reasons and still trigger an upgrade. if tc.wantUpdate { assert.NotNil(unstructuredClient.updatedObject) @@ -410,7 +348,9 @@ func TestIsValidImageUpgrade(t *testing.T) { assert := assert.New(t) upgrader := &KubeCmd{ - log: logger.NewTest(t), + retryInterval: time.Millisecond, + maxAttempts: 5, + log: logger.NewTest(t), } nodeVersion := updatev1alpha1.NodeVersion{ @@ -461,7 +401,9 @@ func TestUpdateK8s(t *testing.T) { assert := assert.New(t) upgrader := &KubeCmd{ - log: logger.NewTest(t), + retryInterval: time.Millisecond, + maxAttempts: 5, + log: logger.NewTest(t), } nodeVersion := updatev1alpha1.NodeVersion{ @@ -658,9 +600,10 @@ func TestApplyJoinConfig(t *testing.T) { kubectl: tc.kubectl, log: logger.NewTest(t), retryInterval: time.Millisecond, + maxAttempts: 5, } - err := cmd.ApplyJoinConfig(context.Background(), tc.newAttestationCfg, []byte{0x11}) + err := cmd.ApplyJoinConfig(t.Context(), tc.newAttestationCfg, []byte{0x11}) if tc.wantErr { assert.Error(err) return @@ -680,6 +623,106 @@ func TestApplyJoinConfig(t *testing.T) { } } +func TestRetryAction(t *testing.T) { + maxAttempts := 3 + + testCases := map[string]struct { + failures int + wantErr bool + }{ + "no failures": { + failures: 0, + }, + "fail once": { + failures: 1, + }, + "fail equal to maxAttempts": { + failures: maxAttempts, + wantErr: true, + }, + "fail more than maxAttempts": { + failures: maxAttempts + 5, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + k := &KubeCmd{ + retryInterval: time.Millisecond, + maxAttempts: maxAttempts, + log: logger.NewTest(t), + } + + errs := map[int]error{} + for idx := range tc.failures { + errs[idx] = assert.AnError + } + + assert := assert.New(t) + + failureCtr := 0 + action := func(context.Context) error { + defer func() { failureCtr++ }() + return errs[failureCtr] + } + + err := k.retryAction(t.Context(), action) + if tc.wantErr { + assert.Error(err) + assert.Equal(min(tc.failures, maxAttempts), failureCtr) + return + } + assert.NoError(err) + assert.Equal(tc.failures, failureCtr-1) + }) + } +} + +func TestExtendClusterConfigCertSANs(t *testing.T) { + ctx := t.Context() + + testCases := map[string]struct { + clusterConfig string + }{ + "kubeadmv1beta3.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta3, + }, + "kubeadmv1beta4.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta4, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + kubectl := &fakeConfigMapClient{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": tc.clusterConfig}}, + }, + } + cmd := &KubeCmd{ + kubectl: kubectl, + log: logger.NewTest(t), + retryInterval: time.Millisecond, + } + + err := cmd.ExtendClusterConfigCertSANs(ctx, []string{"example.com"}) + require.NoError(err) + + cm := kubectl.configMaps["kubeadm-config"] + require.NotNil(cm) + cc := cm.Data["ClusterConfiguration"] + require.NotNil(cc) + // Verify that SAN was added. + assert.Contains(cc, "example.com") + // Verify that config was written in v1beta4, regardless of the version read. + assert.Contains(cc, "kubeadm.k8s.io/v1beta4") + }) + } +} + type fakeUnstructuredClient struct { mock.Mock } @@ -839,3 +882,83 @@ func supportedValidK8sVersions() (res []versions.ValidK8sVersion) { } return } + +var kubeadmClusterConfigurationV1Beta3 = ` +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt + profiling: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + cloud-provider: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + profiling: "false" +` + +var kubeadmClusterConfigurationV1Beta4 = ` +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + - name: kubelet-certificate-authority + value: /etc/kubernetes/pki/ca.crt + - name: profiling + value: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + - name: cloud-provider + value: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + - name: profiling + value: "false" +` diff --git a/internal/constellation/kubecmd/status.go b/internal/constellation/kubecmd/status.go index 9d7ee2a66..328ed38ba 100644 --- a/internal/constellation/kubecmd/status.go +++ b/internal/constellation/kubecmd/status.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubecmd @@ -9,7 +9,7 @@ package kubecmd import ( "fmt" - updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" ) diff --git a/internal/constellation/kubernetes.go b/internal/constellation/kubernetes.go index af038adce..30b553816 100644 --- a/internal/constellation/kubernetes.go +++ b/internal/constellation/kubernetes.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation diff --git a/internal/constellation/serviceaccount.go b/internal/constellation/serviceaccount.go index c88d92a19..9c38c94e6 100644 --- a/internal/constellation/serviceaccount.go +++ b/internal/constellation/serviceaccount.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package constellation diff --git a/internal/constellation/state/BUILD.bazel b/internal/constellation/state/BUILD.bazel index a3cac2577..a60d5c950 100644 --- a/internal/constellation/state/BUILD.bazel +++ b/internal/constellation/state/BUILD.bazel @@ -10,7 +10,8 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/state", visibility = ["//:__subpackages__"], deps = [ - "//internal/cloud/cloudprovider", + "//internal/attestation/variant", + "//internal/encoding", "//internal/file", "//internal/validation", "@cat_dario_mergo//:mergo", @@ -26,13 +27,12 @@ go_test( ], embed = [":state"], deps = [ - "//internal/cloud/cloudprovider", + "//internal/attestation/variant", "//internal/constants", "//internal/file", "@com_github_siderolabs_talos_pkg_machinery//config/encoder", "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@in_gopkg_yaml_v3//:yaml_v3", ], ) diff --git a/internal/constellation/state/state.go b/internal/constellation/state/state.go index ea6b15db5..af902900c 100644 --- a/internal/constellation/state/state.go +++ b/internal/constellation/state/state.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // This binary can be build from siderolabs/talos projects. Located at: @@ -13,13 +13,13 @@ SPDX-License-Identifier: AGPL-3.0-only package state import ( - "encoding/hex" "errors" "fmt" "os" "dario.cat/mergo" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" + "github.com/edgelesssys/constellation/v2/internal/encoding" "github.com/edgelesssys/constellation/v2/internal/file" "github.com/edgelesssys/constellation/v2/internal/validation" ) @@ -76,9 +76,6 @@ type State struct { // description: | // Schema version of this state file. Version string `yaml:"version"` - - // TODO(msanft): Add link to self-managed infrastructure docs once existing. - // description: | // State of the cluster's cloud resources. These values are retrieved during // cluster creation. In the case of self-managed infrastructure, the marked @@ -101,7 +98,7 @@ type ClusterValues struct { OwnerID string `yaml:"ownerID"` // description: | // Salt used to generate the ClusterID on the bootstrapping node. - MeasurementSalt HexBytes `yaml:"measurementSalt"` + MeasurementSalt encoding.HexBytes `yaml:"measurementSalt"` } // Infrastructure describe the state related to the cloud resources of the cluster. @@ -118,7 +115,7 @@ type Infrastructure struct { InClusterEndpoint string `yaml:"inClusterEndpoint"` // description: | // Secret used to authenticate the bootstrapping node. - InitSecret HexBytes `yaml:"initSecret"` + InitSecret encoding.HexBytes `yaml:"initSecret"` // description: | // List of Subject Alternative Names (SANs) to add to the Kubernetes API server certificate. // If no SANs should be added, this field can be left empty. @@ -135,6 +132,9 @@ type Infrastructure struct { // description: | // Values specific to a Constellation cluster running on GCP. GCP *GCP `yaml:"gcp,omitempty"` + // description: | + // Values specific to a Constellation cluster running on OpenStack. + OpenStack *OpenStack `yaml:"openstack,omitempty"` } // GCP describes the infra state related to GCP. @@ -171,6 +171,16 @@ type Azure struct { AttestationURL string `yaml:"attestationURL"` } +// OpenStack describes the infra state related to OpenStack. +type OpenStack struct { + // description: | + // ID of the network + NetworkID string `yaml:"networkID"` + // description: | + // ID of the subnet + SubnetID string `yaml:"subnetID"` +} + // New creates a new cluster state (file). func New() *State { return &State{ @@ -213,7 +223,7 @@ Validate validates the state against the given constraint set and CSP, which can - PreInit, which is the constraint set that should be enforced before "constellation apply" is run. - PostInit, which is the constraint set that should be enforced after "constellation apply" is run. */ -func (s *State) Validate(constraintSet ConstraintSet, csp cloudprovider.Provider) error { +func (s *State) Validate(constraintSet ConstraintSet, variant variant.Variant) error { v := validation.NewValidator() switch constraintSet { @@ -223,11 +233,11 @@ func (s *State) Validate(constraintSet ConstraintSet, csp cloudprovider.Provider }) case PreInit: return v.Validate(s, validation.ValidateOptions{ - OverrideConstraints: s.preInitConstraints, + OverrideConstraints: s.preInitConstraints(variant), }) case PostInit: return v.Validate(s, validation.ValidateOptions{ - OverrideConstraints: s.postInitConstraints(csp), + OverrideConstraints: s.postInitConstraints(variant), }) default: return errors.New("unknown constraint set") @@ -282,128 +292,130 @@ func (s *State) preCreateConstraints() []*validation.Constraint { // // The constraints check if the infrastructure state is valid, and if the cluster values // are empty, which is required for the cluster to initialize correctly. -func (s *State) preInitConstraints() []*validation.Constraint { - return []*validation.Constraint{ - // state version needs to be accepted by the parsing CLI. - validation.OneOf(s.Version, []string{Version1}). - WithFieldTrace(s, &s.Version), - // infrastructure must be valid. - // out-of-cluster endpoint needs to be a valid DNS name or IP address. - validation.Or( - validation.DNSName(s.Infrastructure.ClusterEndpoint). - WithFieldTrace(s, &s.Infrastructure.ClusterEndpoint), - validation.IPAddress(s.Infrastructure.ClusterEndpoint). - WithFieldTrace(s, &s.Infrastructure.ClusterEndpoint), - ), - // in-cluster endpoint needs to be a valid DNS name or IP address. - validation.Or( - validation.DNSName(s.Infrastructure.InClusterEndpoint). - WithFieldTrace(s, &s.Infrastructure.InClusterEndpoint), - validation.IPAddress(s.Infrastructure.InClusterEndpoint). - WithFieldTrace(s, &s.Infrastructure.InClusterEndpoint), - ), - // Node IP Cidr needs to be a valid CIDR range. - validation.CIDR(s.Infrastructure.IPCidrNode). - WithFieldTrace(s, &s.Infrastructure.IPCidrNode), - // UID needs to be filled. - validation.NotEmpty(s.Infrastructure.UID). - WithFieldTrace(s, &s.Infrastructure.UID), - // Name needs to be filled. - validation.NotEmpty(s.Infrastructure.Name). - WithFieldTrace(s, &s.Infrastructure.Name), - // GCP values need to be nil, empty, or valid. - validation.Or( +func (s *State) preInitConstraints(attestation variant.Variant) func() []*validation.Constraint { + return func() []*validation.Constraint { + constraints := []*validation.Constraint{ + // state version needs to be accepted by the parsing CLI. + validation.OneOf(s.Version, []string{Version1}). + WithFieldTrace(s, &s.Version), + // infrastructure must be valid. + // out-of-cluster endpoint needs to be a valid DNS name or IP address. validation.Or( - // nil. - validation.Equal(s.Infrastructure.GCP, nil). - WithFieldTrace(s, &s.Infrastructure.GCP), - // empty. - validation.IfNotNil( - s.Infrastructure.GCP, - func() *validation.Constraint { - return validation.Empty(*s.Infrastructure.GCP). - WithFieldTrace(s, &s.Infrastructure.GCP) - }, + validation.DNSName(s.Infrastructure.ClusterEndpoint). + WithFieldTrace(s, &s.Infrastructure.ClusterEndpoint), + validation.IPAddress(s.Infrastructure.ClusterEndpoint). + WithFieldTrace(s, &s.Infrastructure.ClusterEndpoint), + ), + // in-cluster endpoint needs to be a valid DNS name or IP address. + validation.Or( + validation.DNSName(s.Infrastructure.InClusterEndpoint). + WithFieldTrace(s, &s.Infrastructure.InClusterEndpoint), + validation.IPAddress(s.Infrastructure.InClusterEndpoint). + WithFieldTrace(s, &s.Infrastructure.InClusterEndpoint), + ), + // Node IP Cidr needs to be a valid CIDR range. + validation.CIDR(s.Infrastructure.IPCidrNode). + WithFieldTrace(s, &s.Infrastructure.IPCidrNode), + // UID needs to be filled. + validation.NotEmpty(s.Infrastructure.UID). + WithFieldTrace(s, &s.Infrastructure.UID), + // Name needs to be filled. + validation.NotEmpty(s.Infrastructure.Name). + WithFieldTrace(s, &s.Infrastructure.Name), + // ClusterValues must be empty. + // As the clusterValues struct contains slices, we cannot use the + // Empty constraint on the entire struct. Instead, we need to check + // each field individually. + validation.Empty(s.ClusterValues.ClusterID). + WithFieldTrace(s, &s.ClusterValues.ClusterID), + // ownerID is currently unused as functionality is not implemented + // Therefore, we don't want to validate it + // validation.Empty(s.ClusterValues.OwnerID). + // WithFieldTrace(s, &s.ClusterValues.OwnerID), + validation.EmptySlice(s.ClusterValues.MeasurementSalt). + WithFieldTrace(s, &s.ClusterValues.MeasurementSalt), + } + + switch attestation { + case variant.AzureSEVSNP{}, variant.AzureTDX{}, variant.AzureTrustedLaunch{}: + // Azure values need to be valid after infrastructure creation. + constraints = append(constraints, + // GCP values need to be nil or empty. + validation.Or( + validation.Equal(s.Infrastructure.GCP, nil). + WithFieldTrace(s, &s.Infrastructure.GCP), + validation.IfNotNil( + s.Infrastructure.GCP, + func() *validation.Constraint { + return validation.Empty(s.Infrastructure.GCP). + WithFieldTrace(s, &s.Infrastructure.GCP) + }, + ), ), - ), - // valid. - validation.IfNotNil( - s.Infrastructure.GCP, - func() *validation.Constraint { - return validation.And( - validation.EvaluateAll, - // ProjectID needs to be filled. - validation.NotEmpty(s.Infrastructure.GCP.ProjectID). - WithFieldTrace(s, &s.Infrastructure.GCP.ProjectID), - // Pod IP Cidr needs to be a valid CIDR range. - validation.CIDR(s.Infrastructure.GCP.IPCidrPod). - WithFieldTrace(s, &s.Infrastructure.GCP.IPCidrPod), - ) - }, - ), - ), - // Azure values need to be nil, empty, or valid. - validation.Or( - validation.Or( - // nil. - validation.Equal(s.Infrastructure.Azure, nil). - WithFieldTrace(s, &s.Infrastructure.Azure), - // empty. validation.IfNotNil( s.Infrastructure.Azure, func() *validation.Constraint { return validation.And( validation.EvaluateAll, - validation.Empty(s.Infrastructure.Azure.ResourceGroup). + validation.NotEmpty(s.Infrastructure.Azure.ResourceGroup). WithFieldTrace(s, &s.Infrastructure.Azure.ResourceGroup), - validation.Empty(s.Infrastructure.Azure.SubscriptionID). + validation.NotEmpty(s.Infrastructure.Azure.SubscriptionID). WithFieldTrace(s, &s.Infrastructure.Azure.SubscriptionID), - validation.Empty(s.Infrastructure.Azure.NetworkSecurityGroupName). + validation.NotEmpty(s.Infrastructure.Azure.NetworkSecurityGroupName). WithFieldTrace(s, &s.Infrastructure.Azure.NetworkSecurityGroupName), - validation.Empty(s.Infrastructure.Azure.LoadBalancerName). + validation.NotEmpty(s.Infrastructure.Azure.LoadBalancerName). WithFieldTrace(s, &s.Infrastructure.Azure.LoadBalancerName), - validation.Empty(s.Infrastructure.Azure.UserAssignedIdentity). + validation.NotEmpty(s.Infrastructure.Azure.UserAssignedIdentity). WithFieldTrace(s, &s.Infrastructure.Azure.UserAssignedIdentity), - validation.Empty(s.Infrastructure.Azure.AttestationURL). - WithFieldTrace(s, &s.Infrastructure.Azure.AttestationURL), ) }, ), - ), - // valid. - validation.IfNotNil( - s.Infrastructure.Azure, - func() *validation.Constraint { - return validation.And( - validation.EvaluateAll, - validation.NotEmpty(s.Infrastructure.Azure.ResourceGroup). - WithFieldTrace(s, &s.Infrastructure.Azure.ResourceGroup), - validation.NotEmpty(s.Infrastructure.Azure.SubscriptionID). - WithFieldTrace(s, &s.Infrastructure.Azure.SubscriptionID), - validation.NotEmpty(s.Infrastructure.Azure.NetworkSecurityGroupName). - WithFieldTrace(s, &s.Infrastructure.Azure.NetworkSecurityGroupName), - validation.NotEmpty(s.Infrastructure.Azure.LoadBalancerName). - WithFieldTrace(s, &s.Infrastructure.Azure.LoadBalancerName), - validation.NotEmpty(s.Infrastructure.Azure.UserAssignedIdentity). - WithFieldTrace(s, &s.Infrastructure.Azure.UserAssignedIdentity), - validation.NotEmpty(s.Infrastructure.Azure.AttestationURL). - WithFieldTrace(s, &s.Infrastructure.Azure.AttestationURL), - ) - }, - ), - ), - // ClusterValues must be empty. - // As the clusterValues struct contains slices, we cannot use the - // Empty constraint on the entire struct. Instead, we need to check - // each field individually. - validation.Empty(s.ClusterValues.ClusterID). - WithFieldTrace(s, &s.ClusterValues.ClusterID), - // ownerID is currently unused as functionality is not implemented - // Therefore, we don't want to validate it - // validation.Empty(s.ClusterValues.OwnerID). - // WithFieldTrace(s, &s.ClusterValues.OwnerID), - validation.EmptySlice(s.ClusterValues.MeasurementSalt). - WithFieldTrace(s, &s.ClusterValues.MeasurementSalt), + ) + if attestation.Equal(variant.AzureSEVSNP{}) { + constraints = append(constraints, + validation.IfNotNil( + s.Infrastructure.Azure, + func() *validation.Constraint { + // For SEV-SNP attestation, we require the attestation URL to be set. + return validation.NotEmpty(s.Infrastructure.Azure.AttestationURL). + WithFieldTrace(s, &s.Infrastructure.Azure.AttestationURL) + }, + ), + ) + } + case variant.GCPSEVES{}, variant.GCPSEVSNP{}: + // GCP values need to be valid after infrastructure creation. + constraints = append(constraints, + // Azure values need to be nil or empty. + validation.Or( + validation.Equal(s.Infrastructure.Azure, nil). + WithFieldTrace(s, &s.Infrastructure.Azure), + validation.IfNotNil( + s.Infrastructure.Azure, + func() *validation.Constraint { + return validation.Empty(s.Infrastructure.Azure). + WithFieldTrace(s, &s.Infrastructure.Azure) + }, + ), + ), + validation.IfNotNil( + s.Infrastructure.GCP, + func() *validation.Constraint { + return validation.And( + validation.EvaluateAll, + // ProjectID needs to be filled. + validation.NotEmpty(s.Infrastructure.GCP.ProjectID). + WithFieldTrace(s, &s.Infrastructure.GCP.ProjectID), + // Pod IP Cidr needs to be a valid CIDR range. + validation.CIDR(s.Infrastructure.GCP.IPCidrPod). + WithFieldTrace(s, &s.Infrastructure.GCP.IPCidrPod), + ) + }, + ), + ) + } + + return constraints } } @@ -412,7 +424,7 @@ func (s *State) preInitConstraints() []*validation.Constraint { // // The constraints check if the infrastructure state and cluster state // is valid, so that the cluster can be used correctly. -func (s *State) postInitConstraints(csp cloudprovider.Provider) func() []*validation.Constraint { +func (s *State) postInitConstraints(attestation variant.Variant) func() []*validation.Constraint { return func() []*validation.Constraint { constraints := []*validation.Constraint{ // state version needs to be accepted by the parsing CLI. @@ -455,8 +467,8 @@ func (s *State) postInitConstraints(csp cloudprovider.Provider) func() []*valida WithFieldTrace(s, &s.ClusterValues.MeasurementSalt), } - switch csp { - case cloudprovider.Azure: + switch attestation { + case variant.AzureSEVSNP{}, variant.AzureTDX{}, variant.AzureTrustedLaunch{}: constraints = append(constraints, // GCP values need to be nil or empty. validation.Or( @@ -468,7 +480,8 @@ func (s *State) postInitConstraints(csp cloudprovider.Provider) func() []*valida return validation.Empty(s.Infrastructure.GCP). WithFieldTrace(s, &s.Infrastructure.GCP) }, - )), + ), + ), // Azure values need to be valid. validation.IfNotNil( s.Infrastructure.Azure, @@ -485,13 +498,23 @@ func (s *State) postInitConstraints(csp cloudprovider.Provider) func() []*valida WithFieldTrace(s, &s.Infrastructure.Azure.LoadBalancerName), validation.NotEmpty(s.Infrastructure.Azure.UserAssignedIdentity). WithFieldTrace(s, &s.Infrastructure.Azure.UserAssignedIdentity), - validation.NotEmpty(s.Infrastructure.Azure.AttestationURL). - WithFieldTrace(s, &s.Infrastructure.Azure.AttestationURL), ) }, ), ) - case cloudprovider.GCP: + if attestation.Equal(variant.AzureSEVSNP{}) { + constraints = append(constraints, + validation.IfNotNil( + s.Infrastructure.Azure, + func() *validation.Constraint { + // For SEV-SNP attestation, we require the attestation URL to be set. + return validation.NotEmpty(s.Infrastructure.Azure.AttestationURL). + WithFieldTrace(s, &s.Infrastructure.Azure.AttestationURL) + }, + ), + ) + } + case variant.GCPSEVES{}, variant.GCPSEVSNP{}: constraints = append(constraints, // Azure values need to be nil or empty. validation.Or( @@ -503,7 +526,8 @@ func (s *State) postInitConstraints(csp cloudprovider.Provider) func() []*valida return validation.Empty(s.Infrastructure.Azure). WithFieldTrace(s, &s.Infrastructure.Azure) }, - )), + ), + ), // GCP values need to be valid. validation.IfNotNil( s.Infrastructure.GCP, @@ -554,33 +578,3 @@ func (s *State) postInitConstraints(csp cloudprovider.Provider) func() []*valida func (s *State) Constraints() []*validation.Constraint { return []*validation.Constraint{} } - -// HexBytes is a byte slice that is marshalled to and from a hex string. -type HexBytes []byte - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -func (h *HexBytes) UnmarshalYAML(unmarshal func(any) error) error { - var hexString string - if err := unmarshal(&hexString); err != nil { - // TODO(msanft): Remove with v2.14.0 - // fall back to unmarshalling as a byte slice for backwards compatibility - var oldHexBytes []byte - if err := unmarshal(&oldHexBytes); err != nil { - return fmt.Errorf("unmarshalling hex bytes: %w", err) - } - hexString = hex.EncodeToString(oldHexBytes) - } - - bytes, err := hex.DecodeString(hexString) - if err != nil { - return fmt.Errorf("decoding hex bytes: %w", err) - } - - *h = bytes - return nil -} - -// MarshalYAML implements the yaml.Marshaler interface. -func (h HexBytes) MarshalYAML() (any, error) { - return hex.EncodeToString(h), nil -} diff --git a/internal/constellation/state/state_doc.go b/internal/constellation/state/state_doc.go index 7666c26d9..cf1b99f97 100644 --- a/internal/constellation/state/state_doc.go +++ b/internal/constellation/state/state_doc.go @@ -16,6 +16,7 @@ var ( InfrastructureDoc encoder.Doc GCPDoc encoder.Doc AzureDoc encoder.Doc + OpenStackDoc encoder.Doc ) func init() { @@ -74,7 +75,7 @@ func init() { FieldName: "infrastructure", }, } - InfrastructureDoc.Fields = make([]encoder.Doc, 9) + InfrastructureDoc.Fields = make([]encoder.Doc, 10) InfrastructureDoc.Fields[0].Name = "uid" InfrastructureDoc.Fields[0].Type = "string" InfrastructureDoc.Fields[0].Note = "" @@ -120,6 +121,11 @@ func init() { InfrastructureDoc.Fields[8].Note = "" InfrastructureDoc.Fields[8].Description = "Values specific to a Constellation cluster running on GCP." InfrastructureDoc.Fields[8].Comments[encoder.LineComment] = "Values specific to a Constellation cluster running on GCP." + InfrastructureDoc.Fields[9].Name = "openstack" + InfrastructureDoc.Fields[9].Type = "OpenStack" + InfrastructureDoc.Fields[9].Note = "" + InfrastructureDoc.Fields[9].Description = "Values specific to a Constellation cluster running on OpenStack." + InfrastructureDoc.Fields[9].Comments[encoder.LineComment] = "Values specific to a Constellation cluster running on OpenStack." GCPDoc.Type = "GCP" GCPDoc.Comments[encoder.LineComment] = "GCP describes the infra state related to GCP." @@ -182,6 +188,27 @@ func init() { AzureDoc.Fields[5].Note = "" AzureDoc.Fields[5].Description = "MAA endpoint that can be used as a fallback for veryifying the ID key digests\nin the cluster's attestation report if the enforcement policy is set accordingly.\nCan be left empty otherwise." AzureDoc.Fields[5].Comments[encoder.LineComment] = "MAA endpoint that can be used as a fallback for veryifying the ID key digests" + + OpenStackDoc.Type = "OpenStack" + OpenStackDoc.Comments[encoder.LineComment] = "OpenStack describes the infra state related to OpenStack." + OpenStackDoc.Description = "OpenStack describes the infra state related to OpenStack." + OpenStackDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Infrastructure", + FieldName: "openstack", + }, + } + OpenStackDoc.Fields = make([]encoder.Doc, 2) + OpenStackDoc.Fields[0].Name = "networkID" + OpenStackDoc.Fields[0].Type = "string" + OpenStackDoc.Fields[0].Note = "" + OpenStackDoc.Fields[0].Description = "ID of the network" + OpenStackDoc.Fields[0].Comments[encoder.LineComment] = "ID of the network" + OpenStackDoc.Fields[1].Name = "subnetID" + OpenStackDoc.Fields[1].Type = "string" + OpenStackDoc.Fields[1].Note = "" + OpenStackDoc.Fields[1].Description = "ID of the subnet" + OpenStackDoc.Fields[1].Comments[encoder.LineComment] = "ID of the subnet" } func (_ State) Doc() *encoder.Doc { @@ -204,6 +231,10 @@ func (_ Azure) Doc() *encoder.Doc { return &AzureDoc } +func (_ OpenStack) Doc() *encoder.Doc { + return &OpenStackDoc +} + // GetConfigurationDoc returns documentation for the file ./state_doc.go. func GetConfigurationDoc() *encoder.FileDoc { return &encoder.FileDoc{ @@ -215,6 +246,7 @@ func GetConfigurationDoc() *encoder.FileDoc { &InfrastructureDoc, &GCPDoc, &AzureDoc, + &OpenStackDoc, }, } } diff --git a/internal/constellation/state/state_test.go b/internal/constellation/state/state_test.go index d377be203..402f49681 100644 --- a/internal/constellation/state/state_test.go +++ b/internal/constellation/state/state_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package state @@ -15,7 +15,6 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" ) // defaultState returns a valid default state for testing. @@ -327,97 +326,6 @@ func TestMerge(t *testing.T) { } } -func TestMarshalHexBytes(t *testing.T) { - testCases := map[string]struct { - in HexBytes - expected string - wantErr bool - }{ - "success": { - in: []byte{0xab, 0xcd, 0xef}, - expected: "abcdef\n", - }, - "empty": { - in: []byte{}, - expected: "\"\"\n", - }, - "nil": { - in: nil, - expected: "\"\"\n", - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - - actual, err := yaml.Marshal(tc.in) - - if tc.wantErr { - assert.Error(err) - } else { - assert.NoError(err) - assert.Equal(tc.expected, string(actual)) - } - }) - } -} - -func TestUnmarshalHexBytes(t *testing.T) { - testCases := map[string]struct { - in string - expected HexBytes - wantErr bool - }{ - "success": { - in: "abcdef", - expected: []byte{0xab, 0xcd, 0xef}, - }, - "empty": { - in: "", - expected: nil, - }, - "byte slice compat": { - in: "[0xab, 0xcd, 0xef]", - expected: []byte{0xab, 0xcd, 0xef}, - }, - "byte slice compat 2": { - in: "[00, 12, 34]", - expected: []byte{0x00, 0x0c, 0x22}, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - - var actual HexBytes - err := yaml.Unmarshal([]byte(tc.in), &actual) - - if tc.wantErr { - assert.Error(err) - } else { - assert.NoError(err) - assert.Equal(tc.expected, actual) - } - }) - } -} - -func TestMarshalUnmarshalHexBytes(t *testing.T) { - in := HexBytes{0xab, 0xcd, 0xef} - expected := "abcdef\n" - - actual, err := yaml.Marshal(in) - require.NoError(t, err) - assert.Equal(t, expected, string(actual)) - - var actual2 HexBytes - err = yaml.Unmarshal(actual, &actual2) - require.NoError(t, err) - assert.Equal(t, in, actual2) -} - func TestCreateOrRead(t *testing.T) { testCases := map[string]struct { fs file.Handler diff --git a/internal/constellation/state/validation_test.go b/internal/constellation/state/validation_test.go index c71aca85e..5c5b458fa 100644 --- a/internal/constellation/state/validation_test.go +++ b/internal/constellation/state/validation_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package state @@ -9,7 +9,7 @@ package state import ( "testing" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" + "github.com/edgelesssys/constellation/v2/internal/attestation/variant" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -70,7 +70,7 @@ func TestPreCreateValidation(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - err := tc.stateFile().Validate(PreCreate, cloudprovider.Azure) + err := tc.stateFile().Validate(PreCreate, variant.AzureSEVSNP{}) if tc.wantErr { require.Error(t, err) if tc.errAssertions != nil { @@ -92,6 +92,7 @@ func TestPreInitValidation(t *testing.T) { testCases := map[string]struct { stateFile func() *State + variant variant.Variant wantErr bool errAssertions func(a *assert.Assertions, err error) }{ @@ -172,6 +173,8 @@ func TestPreInitValidation(t *testing.T) { s.Infrastructure.GCP = &GCP{} return s }, + variant: variant.GCPSEVES{}, + wantErr: true, }, "gcp nil": { stateFile: func() *State { @@ -179,6 +182,8 @@ func TestPreInitValidation(t *testing.T) { s.Infrastructure.GCP = nil return s }, + variant: variant.GCPSEVES{}, + wantErr: true, }, "gcp invalid": { stateFile: func() *State { @@ -187,6 +192,7 @@ func TestPreInitValidation(t *testing.T) { return s }, wantErr: true, + variant: variant.GCPSEVES{}, errAssertions: func(a *assert.Assertions, err error) { a.Contains(err.Error(), "validating State.infrastructure.gcp.ipCidrPod: invalid must be a valid CIDR") }, @@ -197,6 +203,8 @@ func TestPreInitValidation(t *testing.T) { s.Infrastructure.Azure = &Azure{} return s }, + variant: variant.AzureSEVSNP{}, + wantErr: true, }, "azure nil": { stateFile: func() *State { @@ -204,6 +212,8 @@ func TestPreInitValidation(t *testing.T) { s.Infrastructure.Azure = nil return s }, + variant: variant.AzureSEVSNP{}, + wantErr: true, }, "azure invalid": { stateFile: func() *State { @@ -215,12 +225,13 @@ func TestPreInitValidation(t *testing.T) { errAssertions: func(a *assert.Assertions, err error) { a.Contains(err.Error(), "validating State.infrastructure.azure.networkSecurityGroupName: must not be empty") }, + variant: variant.AzureSEVSNP{}, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - err := tc.stateFile().Validate(PreInit, cloudprovider.Azure) + err := tc.stateFile().Validate(PreInit, tc.variant) if tc.wantErr { require.Error(t, err) if tc.errAssertions != nil { @@ -236,13 +247,13 @@ func TestPreInitValidation(t *testing.T) { func TestPostInitValidation(t *testing.T) { testCases := map[string]struct { stateFile func() *State - provider cloudprovider.Provider + variant variant.Variant wantErr bool errAssertions func(a *assert.Assertions, err error) }{ "valid": { stateFile: defaultGCPState, - provider: cloudprovider.GCP, + variant: variant.GCPSEVES{}, }, "invalid version": { stateFile: func() *State { @@ -317,22 +328,39 @@ func TestPostInitValidation(t *testing.T) { s := defaultGCPState() return s }, - provider: cloudprovider.GCP, + variant: variant.GCPSEVES{}, }, "azure valid": { stateFile: func() *State { s := defaultAzureState() return s }, - provider: cloudprovider.Azure, + variant: variant.AzureSEVSNP{}, + }, + "azure SEV needs attestation URL": { + stateFile: func() *State { + s := defaultAzureState() + s.Infrastructure.Azure.AttestationURL = "" + return s + }, + variant: variant.AzureSEVSNP{}, + wantErr: true, + }, + "azure TDX does not need attestation URL": { + stateFile: func() *State { + s := defaultAzureState() + s.Infrastructure.Azure.AttestationURL = "" + return s + }, + variant: variant.AzureTDX{}, }, "gcp, azure not nil": { stateFile: func() *State { s := defaultState() return s }, - provider: cloudprovider.GCP, - wantErr: true, + variant: variant.GCPSEVES{}, + wantErr: true, errAssertions: func(a *assert.Assertions, err error) { a.Contains(err.Error(), "must be equal to ") a.Contains(err.Error(), "must be empty") @@ -343,8 +371,8 @@ func TestPostInitValidation(t *testing.T) { s := defaultState() return s }, - provider: cloudprovider.Azure, - wantErr: true, + variant: variant.AzureSEVSNP{}, + wantErr: true, errAssertions: func(a *assert.Assertions, err error) { a.Contains(err.Error(), "must be equal to ") a.Contains(err.Error(), "must be empty") @@ -365,7 +393,7 @@ func TestPostInitValidation(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - err := tc.stateFile().Validate(PostInit, tc.provider) + err := tc.stateFile().Validate(PostInit, tc.variant) if tc.wantErr { require.Error(t, err) if tc.errAssertions != nil { diff --git a/internal/containerimage/containerimage.go b/internal/containerimage/containerimage.go index f5b5fd433..6ed7d20a5 100644 --- a/internal/containerimage/containerimage.go +++ b/internal/containerimage/containerimage.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/crypto/BUILD.bazel b/internal/crypto/BUILD.bazel index 28131c022..0b3e402d9 100644 --- a/internal/crypto/BUILD.bazel +++ b/internal/crypto/BUILD.bazel @@ -6,7 +6,10 @@ go_library( srcs = ["crypto.go"], importpath = "github.com/edgelesssys/constellation/v2/internal/crypto", visibility = ["//:__subpackages__"], - deps = ["@org_golang_x_crypto//hkdf"], + deps = [ + "@org_golang_x_crypto//hkdf", + "@org_golang_x_crypto//ssh", + ], ) go_test( diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 081e25d71..788f4ec89 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package crypto provides functions to for cryptography and random numbers. @@ -9,6 +9,7 @@ package crypto import ( "bytes" + "crypto/ed25519" "crypto/rand" "crypto/sha256" "crypto/x509" @@ -16,8 +17,10 @@ import ( "fmt" "io" "math/big" + "time" "golang.org/x/crypto/hkdf" + "golang.org/x/crypto/ssh" ) const ( @@ -62,6 +65,41 @@ func GenerateRandomBytes(length int) ([]byte, error) { return nonce, nil } +// GenerateEmergencySSHCAKey creates a CA that is used to sign keys for emergency ssh access. +func GenerateEmergencySSHCAKey(seed []byte) (ssh.Signer, error) { + _, priv, err := ed25519.GenerateKey(bytes.NewReader(seed)) + if err != nil { + return nil, err + } + ca, err := ssh.NewSignerFromSigner(priv) + if err != nil { + return nil, err + } + return ca, nil +} + +// GenerateSSHHostCertificate takes a given public key and CA to generate a host certificate. +func GenerateSSHHostCertificate(principals []string, publicKey ssh.PublicKey, ca ssh.Signer) (*ssh.Certificate, error) { + certificate := ssh.Certificate{ + CertType: ssh.HostCert, + ValidPrincipals: principals, + ValidAfter: uint64(time.Now().Unix()), + ValidBefore: ssh.CertTimeInfinity, + Reserved: []byte{}, + Key: publicKey, + KeyId: principals[0], + Permissions: ssh.Permissions{ + CriticalOptions: map[string]string{}, + Extensions: map[string]string{}, + }, + } + if err := certificate.SignCert(rand.Reader, ca); err != nil { + return nil, err + } + + return &certificate, nil +} + // PemToX509Cert takes a list of PEM-encoded certificates, parses the first one and returns it // as an x.509 certificate. func PemToX509Cert(raw []byte) (*x509.Certificate, error) { diff --git a/internal/crypto/crypto_test.go b/internal/crypto/crypto_test.go index be7b0aa77..a99e62dd4 100644 --- a/internal/crypto/crypto_test.go +++ b/internal/crypto/crypto_test.go @@ -1,12 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package crypto import ( + "crypto/ed25519" "crypto/x509" "testing" @@ -17,7 +18,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestDeriveKey(t *testing.T) { @@ -121,6 +122,47 @@ func TestGenerateRandomBytes(t *testing.T) { assert.Len(n3, 16) } +func TestGenerateEmergencySSHCAKey(t *testing.T) { + nullKey := make([]byte, ed25519.SeedSize) + + testCases := map[string]struct { + key []byte + wantErr bool + }{ + "key length = 0": { + key: make([]byte, 0), + wantErr: true, + }, + "valid key": { + key: nullKey, + }, + "nil input": { + key: nil, + wantErr: true, + }, + "long key": { + key: make([]byte, 256), + }, + "key too short": { + key: make([]byte, ed25519.SeedSize-1), + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + _, err := GenerateEmergencySSHCAKey(tc.key) + if tc.wantErr { + assert.Error(err) + } else { + assert.NoError(err) + } + }) + } +} + func TestPemToX509Cert(t *testing.T) { testCases := map[string]struct { pemCert []byte diff --git a/internal/crypto/testvector/testvector.go b/internal/crypto/testvector/testvector.go index 38eac83be..1e02e13c0 100644 --- a/internal/crypto/testvector/testvector.go +++ b/internal/crypto/testvector/testvector.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package testvector provides test vectors for key derivation and crypto functions. diff --git a/internal/cryptsetup/cryptsetup.go b/internal/cryptsetup/cryptsetup.go index a61fb83e8..67e31825a 100644 --- a/internal/cryptsetup/cryptsetup.go +++ b/internal/cryptsetup/cryptsetup.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/cryptsetup/cryptsetup_cgo.go b/internal/cryptsetup/cryptsetup_cgo.go index 555e07dfe..e8ac2e31a 100644 --- a/internal/cryptsetup/cryptsetup_cgo.go +++ b/internal/cryptsetup/cryptsetup_cgo.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cryptsetup diff --git a/internal/cryptsetup/cryptsetup_cross.go b/internal/cryptsetup/cryptsetup_cross.go index df1a30790..325a86be4 100644 --- a/internal/cryptsetup/cryptsetup_cross.go +++ b/internal/cryptsetup/cryptsetup_cross.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cryptsetup diff --git a/internal/encoding/BUILD.bazel b/internal/encoding/BUILD.bazel new file mode 100644 index 000000000..97d3fd903 --- /dev/null +++ b/internal/encoding/BUILD.bazel @@ -0,0 +1,20 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "encoding", + srcs = ["encoding.go"], + importpath = "github.com/edgelesssys/constellation/v2/internal/encoding", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "encoding_test", + srcs = ["encoding_test.go"], + embed = [":encoding"], + deps = [ + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go new file mode 100644 index 000000000..c1fed1815 --- /dev/null +++ b/internal/encoding/encoding.go @@ -0,0 +1,71 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +// Package encoding provides data types and functions for JSON or YAML encoding/decoding. +package encoding + +import ( + "encoding/hex" + "encoding/json" + "fmt" +) + +// HexBytes is a byte slice that is marshalled to and from a hex string. +type HexBytes []byte + +// String returns the hex encoded string representation of the byte slice. +func (h HexBytes) String() string { + return hex.EncodeToString(h) +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (h *HexBytes) UnmarshalJSON(data []byte) error { + var hexString string + if err := json.Unmarshal(data, &hexString); err != nil { + return err + } + // special case to stay consistent with yaml unmarshaler: + // on empty string, unmarshal to nil + if hexString == "" { + *h = nil + return nil + } + return h.unmarshal(hexString) +} + +// MarshalJSON implements the json.Marshaler interface. +func (h HexBytes) MarshalJSON() ([]byte, error) { + return json.Marshal(h.String()) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (h *HexBytes) UnmarshalYAML(unmarshal func(any) error) error { + var hexString string + if err := unmarshal(&hexString); err != nil { + // compatibility mode for old state file format: + // fall back to unmarshalling as a byte slice for backwards compatibility + var oldHexBytes []byte + if err := unmarshal(&oldHexBytes); err != nil { + return fmt.Errorf("unmarshalling hex bytes: %w", err) + } + hexString = hex.EncodeToString(oldHexBytes) + } + return h.unmarshal(hexString) +} + +// MarshalYAML implements the yaml.Marshaler interface. +func (h HexBytes) MarshalYAML() (any, error) { + return h.String(), nil +} + +func (h *HexBytes) unmarshal(hexString string) error { + bytes, err := hex.DecodeString(hexString) + if err != nil { + return fmt.Errorf("decoding hex bytes: %w", err) + } + *h = bytes + return nil +} diff --git a/internal/encoding/encoding_test.go b/internal/encoding/encoding_test.go new file mode 100644 index 000000000..54600e88f --- /dev/null +++ b/internal/encoding/encoding_test.go @@ -0,0 +1,136 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package encoding + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestMarshalHexBytes(t *testing.T) { + testCases := map[string]struct { + in HexBytes + expectedJSON string + expectedYAML string + wantErr bool + }{ + "success": { + in: []byte{0xab, 0xcd, 0xef}, + expectedJSON: "\"abcdef\"", + expectedYAML: "abcdef\n", + }, + "empty": { + in: []byte{}, + expectedJSON: "\"\"", + expectedYAML: "\"\"\n", + }, + "nil": { + in: nil, + expectedJSON: "\"\"", + expectedYAML: "\"\"\n", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + actualYAML, errYAML := yaml.Marshal(tc.in) + actualJSON, errJSON := json.Marshal(tc.in) + + if tc.wantErr { + assert.Error(errYAML) + assert.Error(errJSON) + return + } + assert.NoError(errYAML) + assert.NoError(errJSON) + assert.Equal(tc.expectedYAML, string(actualYAML), "yaml") + assert.Equal(tc.expectedJSON, string(actualJSON), "json") + }) + } +} + +func TestUnmarshalHexBytes(t *testing.T) { + testCases := map[string]struct { + yamlString string + jsonString string + expected HexBytes + wantErr bool + }{ + "success": { + yamlString: "abcdef", + jsonString: "\"abcdef\"", + expected: []byte{0xab, 0xcd, 0xef}, + }, + "empty": { + yamlString: "", + jsonString: "\"\"", + expected: nil, + }, + "byte slice compat": { + yamlString: "[0xab, 0xcd, 0xef]", + jsonString: "\"abcdef\"", // no backwards compatibility since we never used this format for json + expected: []byte{0xab, 0xcd, 0xef}, + }, + "byte slice compat 2": { + yamlString: "[00, 12, 34]", + jsonString: "\"000c22\"", // no backwards compatibility since we never used this format for json + expected: []byte{0x00, 0x0c, 0x22}, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + var actualYAML HexBytes + errYAML := yaml.Unmarshal([]byte(tc.yamlString), &actualYAML) + var actualJSON HexBytes + errJSON := json.Unmarshal([]byte(tc.jsonString), &actualJSON) + + if tc.wantErr { + assert.Error(errYAML) + assert.Error(errJSON) + return + } + assert.NoError(errYAML) + assert.NoError(errJSON) + assert.Equal(tc.expected, actualYAML, "yaml") + assert.Equal(tc.expected, actualJSON, "json") + }) + } +} + +func TestMarshalUnmarshalHexBytes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + in := HexBytes{0xab, 0xcd, 0xef} + expectedJSON := "\"abcdef\"" + expectedYAML := "abcdef\n" + + actualJSON, err := json.Marshal(in) + require.NoError(err) + assert.Equal(expectedJSON, string(actualJSON)) + actualYAML, err := yaml.Marshal(in) + require.NoError(err) + assert.Equal(expectedYAML, string(actualYAML)) + + var actualJSON2 HexBytes + err = json.Unmarshal(actualJSON, &actualJSON2) + require.NoError(err) + assert.Equal(in, actualJSON2) + var actualYAML2 HexBytes + err = yaml.Unmarshal(actualYAML, &actualYAML2) + require.NoError(err) + assert.Equal(in, actualYAML2) +} diff --git a/internal/file/file.go b/internal/file/file.go index 84e0104b2..8bfb9ecbe 100644 --- a/internal/file/file.go +++ b/internal/file/file.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -233,8 +233,8 @@ func (h *Handler) CopyFile(src, dst string, opts ...Option) error { } // RenameFile renames a file, overwriting any existing file at the destination. -func (h *Handler) RenameFile(old, new string) error { - return h.fs.Rename(old, new) +func (h *Handler) RenameFile(a, b string) error { + return h.fs.Rename(a, b) } // IsEmpty returns true if the given directory is empty. diff --git a/internal/file/file_test.go b/internal/file/file_test.go index a95c439df..e18341a18 100644 --- a/internal/file/file_test.go +++ b/internal/file/file_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package file @@ -21,7 +21,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestWrite(t *testing.T) { diff --git a/internal/grpc/atlscredentials/BUILD.bazel b/internal/grpc/atlscredentials/BUILD.bazel index 8f16bd02f..c69ef3bda 100644 --- a/internal/grpc/atlscredentials/BUILD.bazel +++ b/internal/grpc/atlscredentials/BUILD.bazel @@ -21,7 +21,7 @@ go_test( "//internal/atls", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//test/bufconn", "@org_uber_go_goleak//:goleak", ], diff --git a/internal/grpc/atlscredentials/atlscredentials.go b/internal/grpc/atlscredentials/atlscredentials.go index 949f9af41..cb1c1dca5 100644 --- a/internal/grpc/atlscredentials/atlscredentials.go +++ b/internal/grpc/atlscredentials/atlscredentials.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package atlscredentials handles creation of TLS credentials for attested TLS (ATLS). diff --git a/internal/grpc/atlscredentials/atlscredentials_test.go b/internal/grpc/atlscredentials/atlscredentials_test.go index 2fe3fbf54..d03a03e94 100644 --- a/internal/grpc/atlscredentials/atlscredentials_test.go +++ b/internal/grpc/atlscredentials/atlscredentials_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package atlscredentials @@ -25,7 +25,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestATLSCredentials(t *testing.T) { @@ -66,14 +66,14 @@ func TestATLSCredentials(t *testing.T) { go func() { var err error defer func() { errChan <- err }() - conn, err := grpc.DialContext(context.Background(), "", grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + conn, err := grpc.NewClient("192.0.2.1", grpc.WithContextDialer(func(_ context.Context, _ string) (net.Conn, error) { return lis.Dial() }), grpc.WithTransportCredentials(clientCreds)) require.NoError(err) defer conn.Close() client := initproto.NewAPIClient(conn) - _, err = client.Init(context.Background(), &initproto.InitRequest{}) + _, err = client.Init(t.Context(), &initproto.InitRequest{}) }() } diff --git a/internal/grpc/dialer/BUILD.bazel b/internal/grpc/dialer/BUILD.bazel index b8428ba2d..9dc1aaf8f 100644 --- a/internal/grpc/dialer/BUILD.bazel +++ b/internal/grpc/dialer/BUILD.bazel @@ -9,7 +9,7 @@ go_library( deps = [ "//internal/atls", "//internal/grpc/atlscredentials", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//credentials/insecure", ], ) @@ -25,7 +25,7 @@ go_test( "//internal/grpc/testdialer", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//interop/grpc_testing", "@org_uber_go_goleak//:goleak", ], diff --git a/internal/grpc/dialer/dialer.go b/internal/grpc/dialer/dialer.go index 8c42f4041..b81ad1fe4 100644 --- a/internal/grpc/dialer/dialer.go +++ b/internal/grpc/dialer/dialer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package dialer provides a grpc dialer that can be used to create grpc client connections with different levels of ATLS encryption / verification. @@ -34,14 +34,14 @@ func New(issuer atls.Issuer, validator atls.Validator, netDialer NetDialer) *Dia } // Dial creates a new grpc client connection to the given target using the atls validator. -func (d *Dialer) Dial(ctx context.Context, target string) (*grpc.ClientConn, error) { +func (d *Dialer) Dial(target string) (*grpc.ClientConn, error) { var validators []atls.Validator if d.validator != nil { validators = append(validators, d.validator) } credentials := atlscredentials.New(d.issuer, validators) - return grpc.DialContext(ctx, target, + return grpc.NewClient(target, d.grpcWithDialer(), grpc.WithTransportCredentials(credentials), ) @@ -49,24 +49,27 @@ func (d *Dialer) Dial(ctx context.Context, target string) (*grpc.ClientConn, err // DialInsecure creates a new grpc client connection to the given target without using encryption or verification. // Only use this method when using another kind of encryption / verification (VPN, etc). -func (d *Dialer) DialInsecure(ctx context.Context, target string) (*grpc.ClientConn, error) { - return grpc.DialContext(ctx, target, +func (d *Dialer) DialInsecure(target string) (*grpc.ClientConn, error) { + return grpc.NewClient(target, d.grpcWithDialer(), grpc.WithTransportCredentials(insecure.NewCredentials()), ) } // DialNoVerify creates a new grpc client connection to the given target without verifying the server's attestation. -func (d *Dialer) DialNoVerify(ctx context.Context, target string) (*grpc.ClientConn, error) { +func (d *Dialer) DialNoVerify(target string) (*grpc.ClientConn, error) { credentials := atlscredentials.New(nil, nil) - return grpc.DialContext(ctx, target, + return grpc.NewClient(target, d.grpcWithDialer(), grpc.WithTransportCredentials(credentials), ) } func (d *Dialer) grpcWithDialer() grpc.DialOption { + if d.netDialer == nil { + return grpc.EmptyDialOption{} + } return grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { return d.netDialer.DialContext(ctx, "tcp", addr) }) diff --git a/internal/grpc/dialer/dialer_test.go b/internal/grpc/dialer/dialer_test.go index 4e152bd56..6520cf08d 100644 --- a/internal/grpc/dialer/dialer_test.go +++ b/internal/grpc/dialer/dialer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package dialer @@ -22,48 +22,48 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestDial(t *testing.T) { testCases := map[string]struct { tls bool - dialFn func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) + dialFn func(dialer *Dialer, target string) (*grpc.ClientConn, error) wantErr bool }{ "Dial with tls on server works": { tls: true, - dialFn: func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) { - return dialer.Dial(ctx, target) + dialFn: func(dialer *Dialer, target string) (*grpc.ClientConn, error) { + return dialer.Dial(target) }, }, "Dial without tls on server fails": { - dialFn: func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) { - return dialer.Dial(ctx, target) + dialFn: func(dialer *Dialer, target string) (*grpc.ClientConn, error) { + return dialer.Dial(target) }, wantErr: true, }, "DialNoVerify with tls on server works": { tls: true, - dialFn: func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) { - return dialer.DialNoVerify(ctx, target) + dialFn: func(dialer *Dialer, target string) (*grpc.ClientConn, error) { + return dialer.DialNoVerify(target) }, }, "DialNoVerify without tls on server fails": { - dialFn: func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) { - return dialer.DialNoVerify(ctx, target) + dialFn: func(dialer *Dialer, target string) (*grpc.ClientConn, error) { + return dialer.DialNoVerify(target) }, wantErr: true, }, "DialInsecure without tls on server works": { - dialFn: func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) { - return dialer.DialInsecure(ctx, target) + dialFn: func(dialer *Dialer, target string) (*grpc.ClientConn, error) { + return dialer.DialInsecure(target) }, }, "DialInsecure with tls on server fails": { tls: true, - dialFn: func(dialer *Dialer, ctx context.Context, target string) (*grpc.ClientConn, error) { - return dialer.DialInsecure(ctx, target) + dialFn: func(dialer *Dialer, target string) (*grpc.ClientConn, error) { + return dialer.DialInsecure(target) }, wantErr: true, }, @@ -81,12 +81,12 @@ func TestDial(t *testing.T) { grpc_testing.RegisterTestServiceServer(server, api) go server.Serve(netDialer.GetListener("192.0.2.1:1234")) defer server.Stop() - conn, err := tc.dialFn(dialer, context.Background(), "192.0.2.1:1234") + conn, err := tc.dialFn(dialer, "192.0.2.1:1234") require.NoError(err) defer conn.Close() client := grpc_testing.NewTestServiceClient(conn) - _, err = client.EmptyCall(context.Background(), &grpc_testing.Empty{}) + _, err = client.EmptyCall(t.Context(), &grpc_testing.Empty{}) if tc.wantErr { assert.Error(err) diff --git a/internal/grpc/grpclog/grpclog.go b/internal/grpc/grpclog/grpclog.go index aceb4af46..c92a4f7f4 100644 --- a/internal/grpc/grpclog/grpclog.go +++ b/internal/grpc/grpclog/grpclog.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // grpclog provides a logging utilities for gRPC. @@ -9,6 +9,7 @@ package grpclog import ( "context" + "fmt" "sync" "google.golang.org/grpc/connectivity" @@ -30,15 +31,15 @@ func LogStateChangesUntilReady(ctx context.Context, conn getStater, log debugLog go func() { defer wg.Done() state := conn.GetState() - log.Debugf("Connection state started as %s", state) + log.Debug(fmt.Sprintf("Connection state started as %q", state)) for ; state != connectivity.Ready && conn.WaitForStateChange(ctx, state); state = conn.GetState() { - log.Debugf("Connection state changed to %s", state) + log.Debug(fmt.Sprintf("Connection state changed to %q", state)) } if state == connectivity.Ready { - log.Debugf("Connection ready") + log.Debug("Connection ready") isReadyCallback() } else { - log.Debugf("Connection state ended with %s", state) + log.Debug(fmt.Sprintf("Connection state ended with %q", state)) } }() } @@ -49,5 +50,5 @@ type getStater interface { } type debugLog interface { - Debugf(format string, args ...any) + Debug(msg string, args ...any) } diff --git a/internal/grpc/grpclog/grpclog_test.go b/internal/grpc/grpclog/grpclog_test.go index 7460de7d9..caebc0770 100644 --- a/internal/grpc/grpclog/grpclog_test.go +++ b/internal/grpc/grpclog/grpclog_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // grpclog provides a logging utilities for gRPC. @@ -9,7 +9,6 @@ package grpclog import ( "context" - "fmt" "sync" "testing" @@ -34,8 +33,8 @@ func TestLogStateChanges(t *testing.T) { }, assert: func(t *testing.T, lg *spyLog, isReadyCallbackCalled bool) { require.Len(t, lg.msgs, 3) - assert.Equal(t, "Connection state started as CONNECTING", lg.msgs[0]) - assert.Equal(t, "Connection state changed to CONNECTING", lg.msgs[1]) + assert.Equal(t, "Connection state started as \"CONNECTING\"", lg.msgs[0]) + assert.Equal(t, "Connection state changed to \"CONNECTING\"", lg.msgs[1]) assert.Equal(t, "Connection ready", lg.msgs[2]) assert.True(t, isReadyCallbackCalled) }, @@ -50,7 +49,7 @@ func TestLogStateChanges(t *testing.T) { }, assert: func(t *testing.T, lg *spyLog, isReadyCallbackCalled bool) { require.Len(t, lg.msgs, 2) - assert.Equal(t, "Connection state started as READY", lg.msgs[0]) + assert.Equal(t, "Connection state started as \"READY\"", lg.msgs[0]) assert.Equal(t, "Connection ready", lg.msgs[1]) assert.True(t, isReadyCallbackCalled) }, @@ -65,8 +64,8 @@ func TestLogStateChanges(t *testing.T) { }, assert: func(t *testing.T, lg *spyLog, isReadyCallbackCalled bool) { require.Len(t, lg.msgs, 2) - assert.Equal(t, "Connection state started as CONNECTING", lg.msgs[0]) - assert.Equal(t, "Connection state ended with CONNECTING", lg.msgs[1]) + assert.Equal(t, "Connection state started as \"CONNECTING\"", lg.msgs[0]) + assert.Equal(t, "Connection state ended with \"CONNECTING\"", lg.msgs[1]) assert.False(t, isReadyCallbackCalled) }, }, @@ -77,7 +76,7 @@ func TestLogStateChanges(t *testing.T) { var wg sync.WaitGroup isReadyCallbackCalled := false - LogStateChangesUntilReady(context.Background(), tc.conn, logger, &wg, func() { isReadyCallbackCalled = true }) + LogStateChangesUntilReady(t.Context(), tc.conn, logger, &wg, func() { isReadyCallbackCalled = true }) wg.Wait() tc.assert(t, logger, isReadyCallbackCalled) }) @@ -88,8 +87,8 @@ type spyLog struct { msgs []string } -func (f *spyLog) Debugf(format string, args ...any) { - f.msgs = append(f.msgs, fmt.Sprintf(format, args...)) +func (f *spyLog) Debug(msg string, _ ...any) { + f.msgs = append(f.msgs, msg) } type stubConn struct { diff --git a/internal/grpc/retry/retry.go b/internal/grpc/retry/retry.go index 9d03279a4..3a0f1724b 100644 --- a/internal/grpc/retry/retry.go +++ b/internal/grpc/retry/retry.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package retry provides functions to check if a gRPC error is retryable. @@ -16,9 +16,10 @@ import ( ) const ( - authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"` - authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp` - authHandshakeErr = `connection error: desc = "transport: authentication handshake failed` + authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"` + authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp` + authHandshakeErr = `connection error: desc = "transport: authentication handshake failed` + authHandshakeDeadlineExceededErr = `connection error: desc = "transport: authentication handshake failed: context deadline exceeded` ) // grpcErr is the error type that is returned by the grpc client. @@ -57,6 +58,11 @@ func ServiceIsUnavailable(err error) bool { return true } + // retry if the handshake deadline was exceeded + if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) { + return true + } + return !strings.HasPrefix(statusErr.Message(), authHandshakeErr) } @@ -76,6 +82,11 @@ func LoadbalancerIsNotReady(err error) bool { return false } + // retry if the handshake deadline was exceeded + if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) { + return true + } + // retry if GCP proxy LB isn't fully available yet return strings.HasPrefix(statusErr.Message(), authReadTCPErr) } diff --git a/internal/grpc/retry/retry_test.go b/internal/grpc/retry/retry_test.go index a1b44dce4..b6ad075ed 100644 --- a/internal/grpc/retry/retry_test.go +++ b/internal/grpc/retry/retry_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package retry @@ -43,6 +43,10 @@ func TestServiceIsUnavailable(t *testing.T) { err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`), wantUnavailable: true, }, + "handshake deadline exceeded error": { + err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`), + wantUnavailable: true, + }, "wrapped error": { err: fmt.Errorf("some wrapping: %w", status.Error(codes.Unavailable, "error")), wantUnavailable: true, @@ -82,6 +86,10 @@ func TestLoadbalancerIsNotReady(t *testing.T) { err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`), wantNotReady: true, }, + "handshake deadline exceeded error": { + err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`), + wantNotReady: true, + }, "normal unavailable error": { err: status.Error(codes.Unavailable, "error"), }, diff --git a/internal/grpc/testdialer/testdialer.go b/internal/grpc/testdialer/testdialer.go index e6771903f..d95c2be69 100644 --- a/internal/grpc/testdialer/testdialer.go +++ b/internal/grpc/testdialer/testdialer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package testdialer provides a fake dialer for testing. diff --git a/internal/imagefetcher/imagefetcher.go b/internal/imagefetcher/imagefetcher.go index 044870f55..827adfc89 100644 --- a/internal/imagefetcher/imagefetcher.go +++ b/internal/imagefetcher/imagefetcher.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -18,6 +18,7 @@ import ( "fmt" "io/fs" "regexp" + "strings" "github.com/edgelesssys/constellation/v2/internal/api/fetcher" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" @@ -52,10 +53,6 @@ func (f *Fetcher) FetchReference(ctx context.Context, return "", fmt.Errorf("parsing config image short path: %w", err) } - if useMarketplaceImage { - return buildMarketplaceImage(ver, provider) - } - imgInfoReq := versionsapi.ImageInfo{ Ref: ver.Ref(), Stream: ver.Stream(), @@ -85,21 +82,60 @@ func (f *Fetcher) FetchReference(ctx context.Context, return "", fmt.Errorf("validating image info file: %w", err) } + if useMarketplaceImage { + return buildMarketplaceImage(marketplaceImagePayload{ + ver: ver, + provider: provider, + attestationVariant: attestationVariant, + imgInfo: imgInfo, + filters: filters(provider, region), + }) + } + return getReferenceFromImageInfo(provider, attestationVariant.String(), imgInfo, filters(provider, region)...) } +// marketplaceImagePayload is a helper struct to pass around the required information to build a marketplace image URI. +type marketplaceImagePayload struct { + ver versionsapi.Version + provider cloudprovider.Provider + attestationVariant variant.Variant + imgInfo versionsapi.ImageInfo + filters []filter +} + // buildMarketplaceImage returns a marketplace image URI for the given CSP and version. -func buildMarketplaceImage(ver versionsapi.Version, provider cloudprovider.Provider) (string, error) { - sv, err := semver.New(ver.Version()) +func buildMarketplaceImage(payload marketplaceImagePayload) (string, error) { + sv, err := semver.New(payload.ver.Version()) if err != nil { return "", fmt.Errorf("parsing image version: %w", err) } - switch provider { + if sv.Prerelease() != "" && payload.provider != cloudprovider.OpenStack { + return "", fmt.Errorf("marketplace images are not supported for prerelease versions") + } + + switch payload.provider { case cloudprovider.Azure: + // For Azure, multiple fields of information are required to use marketplace images, + // so we pack them in a custom URI. return mpimage.NewAzureMarketplaceImage(sv).URI(), nil + case cloudprovider.GCP: + // For GCP, we just need to replace the GCP project name (constellation-images) to the public project that + // hosts the marketplace images (mpi-edgeless-systems-public). + imageRef, err := getReferenceFromImageInfo(payload.provider, payload.attestationVariant.String(), payload.imgInfo, payload.filters...) + if err != nil { + return "", fmt.Errorf("getting image reference: %w", err) + } + return strings.Replace(imageRef, "constellation-images", "mpi-edgeless-systems-public", 1), nil + case cloudprovider.AWS: + // For AWS, we use the AMI alias, which just needs the version and infers the rest transparently. + return fmt.Sprintf("resolve:ssm:/aws/service/marketplace/prod-77ylkenlkgufs/%s", payload.imgInfo.Version), nil + case cloudprovider.OpenStack: + // For OpenStack / STACKIT, we use the image reference directly. + return getReferenceFromImageInfo(payload.provider, payload.attestationVariant.String(), payload.imgInfo, payload.filters...) default: - return "", fmt.Errorf("marketplace images are not supported for csp %s", provider.String()) + return "", fmt.Errorf("marketplace images are not supported for csp %s", payload.provider.String()) } } diff --git a/internal/imagefetcher/imagefetcher_test.go b/internal/imagefetcher/imagefetcher_test.go index 2146e06d6..e60443ccc 100644 --- a/internal/imagefetcher/imagefetcher_test.go +++ b/internal/imagefetcher/imagefetcher_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package imagefetcher @@ -24,7 +24,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestGetReference(t *testing.T) { @@ -256,7 +256,7 @@ func TestFetchReference(t *testing.T) { fs: af, } - reference, err := fetcher.FetchReference(context.Background(), tc.provider, variant.Dummy{}, + reference, err := fetcher.FetchReference(t.Context(), tc.provider, variant.Dummy{}, tc.image, "someRegion", false) if tc.wantErr { diff --git a/internal/imagefetcher/raw.go b/internal/imagefetcher/raw.go index 593b0d9e7..1375fdca7 100644 --- a/internal/imagefetcher/raw.go +++ b/internal/imagefetcher/raw.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package imagefetcher diff --git a/internal/imagefetcher/raw_test.go b/internal/imagefetcher/raw_test.go index e2bbd8b9d..86a44ab88 100644 --- a/internal/imagefetcher/raw_test.go +++ b/internal/imagefetcher/raw_test.go @@ -1,14 +1,13 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package imagefetcher import ( "bytes" - "context" "io" "net/http" "os" @@ -91,7 +90,7 @@ func TestDownloadWithProgress(t *testing.T) { fs: fs, } var outBuffer bytes.Buffer - err := downloader.downloadWithProgress(context.Background(), &outBuffer, false, tc.source, "someVersion.raw") + err := downloader.downloadWithProgress(t.Context(), &outBuffer, false, tc.source, "someVersion.raw") if tc.wantErr { assert.Error(err) return @@ -167,7 +166,7 @@ func TestDownload(t *testing.T) { fs: fs, } var outBuffer bytes.Buffer - gotDestination, err := downloader.Download(context.Background(), &outBuffer, false, tc.source, "someVersion") + gotDestination, err := downloader.Download(t.Context(), &outBuffer, false, tc.source, "someVersion") if tc.wantErr { assert.Error(err) return diff --git a/internal/installer/installer.go b/internal/installer/installer.go index dd26ea12e..324815b74 100644 --- a/internal/installer/installer.go +++ b/internal/installer/installer.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package installer provides functionality to install binary components of supported kubernetes versions. diff --git a/internal/installer/installer_test.go b/internal/installer/installer_test.go index dead113c0..517a070de 100644 --- a/internal/installer/installer_test.go +++ b/internal/installer/installer_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package installer @@ -80,7 +80,7 @@ func TestInstall(t *testing.T) { wantFiles: map[string][]byte{"/destination": []byte("file-contents")}, }, "download fails": { - server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + server: newHTTPBufconnServer(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }), component: &components.Component{ Url: serverURL, Hash: "sha256:abc", @@ -89,7 +89,7 @@ func TestInstall(t *testing.T) { wantErr: true, }, "dataurl works": { - server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + server: newHTTPBufconnServer(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }), component: &components.Component{ Url: "data:text/plain,file-contents", Hash: "", @@ -98,7 +98,7 @@ func TestInstall(t *testing.T) { wantFiles: map[string][]byte{"/destination": []byte("file-contents")}, }, "broken dataurl fails": { - server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + server: newHTTPBufconnServer(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }), component: &components.Component{ Url: "data:file-contents", Hash: "", @@ -129,10 +129,10 @@ func TestInstall(t *testing.T) { fs: &afero.Afero{Fs: afero.NewMemMapFs()}, hClient: &hClient, clock: testclock.NewFakeClock(time.Time{}), - retriable: func(err error) bool { return false }, + retriable: func(_ error) bool { return false }, } - err := inst.Install(context.Background(), tc.component) + err := inst.Install(t.Context(), tc.component) if tc.wantErr { assert.Error(err) return @@ -340,7 +340,7 @@ func TestRetryDownloadToTempDir(t *testing.T) { } // abort retryDownloadToTempDir in some test cases by using the context - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(t.Context()) defer cancel() wg := sync.WaitGroup{} @@ -388,7 +388,7 @@ func TestDownloadToTempDir(t *testing.T) { wantFile: []byte("file-contents"), }, "download fails": { - server: newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }), + server: newHTTPBufconnServer(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }), wantErr: true, }, "creating temp file fails on RO fs": { @@ -397,7 +397,7 @@ func TestDownloadToTempDir(t *testing.T) { wantErr: true, }, "content length mismatch": { - server: newHTTPBufconnServer(func(writer http.ResponseWriter, request *http.Request) { + server: newHTTPBufconnServer(func(writer http.ResponseWriter, _ *http.Request) { writer.Header().Set("Content-Length", "1337") writer.WriteHeader(200) }), @@ -429,7 +429,7 @@ func TestDownloadToTempDir(t *testing.T) { fs: &afero.Afero{Fs: afs}, hClient: &hClient, } - path, err := inst.downloadToTempDir(context.Background(), "http://server/path") + path, err := inst.downloadToTempDir(t.Context(), "http://server/path") if tc.wantErr { assert.Error(err) return @@ -607,7 +607,7 @@ func newHTTPBufconnServer(handlerFunc http.HandlerFunc) httpBufconnServer { } func newHTTPBufconnServerWithBody(body []byte) httpBufconnServer { - return newHTTPBufconnServer(func(writer http.ResponseWriter, request *http.Request) { + return newHTTPBufconnServer(func(writer http.ResponseWriter, _ *http.Request) { if _, err := writer.Write(body); err != nil { panic(err) } @@ -615,7 +615,7 @@ func newHTTPBufconnServerWithBody(body []byte) httpBufconnServer { } func newHTTPBufconnServerWithState(state chan int, body []byte) httpBufconnServer { - return newHTTPBufconnServer(func(w http.ResponseWriter, r *http.Request) { + return newHTTPBufconnServer(func(w http.ResponseWriter, _ *http.Request) { switch <-state { case 500: w.WriteHeader(500) diff --git a/internal/kms/config/config.go b/internal/kms/config/config.go index 5af6d3e39..92f54979e 100644 --- a/internal/kms/config/config.go +++ b/internal/kms/config/config.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package config provides configuration constants for the KeyService. diff --git a/internal/kms/kms/aws/aws.go b/internal/kms/kms/aws/aws.go index e47cbb9da..9efe03a75 100644 --- a/internal/kms/kms/aws/aws.go +++ b/internal/kms/kms/aws/aws.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package aws implements a KMS backend for AWS KMS. diff --git a/internal/kms/kms/azure/azure.go b/internal/kms/kms/azure/azure.go index 64deec26e..abbf34ed2 100644 --- a/internal/kms/kms/azure/azure.go +++ b/internal/kms/kms/azure/azure.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package azure implements KMS backends for Azure Key Vault and Azure managed HSM. diff --git a/internal/kms/kms/cluster/cluster.go b/internal/kms/kms/cluster/cluster.go index 6ade22f40..a9bc0bab2 100644 --- a/internal/kms/kms/cluster/cluster.go +++ b/internal/kms/kms/cluster/cluster.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kms/kms/cluster/cluster_test.go b/internal/kms/kms/cluster/cluster_test.go index acfffc56b..f276f096e 100644 --- a/internal/kms/kms/cluster/cluster_test.go +++ b/internal/kms/kms/cluster/cluster_test.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package cluster import ( - "context" "strings" "testing" @@ -18,7 +17,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestClusterKMS(t *testing.T) { @@ -29,7 +28,7 @@ func TestClusterKMS(t *testing.T) { require.NoError(err) keyLower, err := kms.GetDEK( - context.Background(), + t.Context(), strings.ToLower(testVector.InfoPrefix+testVector.Info), int(testVector.Length), ) @@ -38,7 +37,7 @@ func TestClusterKMS(t *testing.T) { // output of the KMS should be case sensitive keyUpper, err := kms.GetDEK( - context.Background(), + t.Context(), strings.ToUpper(testVector.InfoPrefix+testVector.Info), int(testVector.Length), ) @@ -105,7 +104,7 @@ func TestVectorsHKDF(t *testing.T) { } require.NoError(err) - out, err := kms.GetDEK(context.Background(), tc.dekID, int(tc.dekSize)) + out, err := kms.GetDEK(t.Context(), tc.dekID, int(tc.dekSize)) require.NoError(err) assert.Equal(tc.wantKey, out) }) diff --git a/internal/kms/kms/gcp/gcp.go b/internal/kms/kms/gcp/gcp.go index dfbdef9af..30a02449d 100644 --- a/internal/kms/kms/gcp/gcp.go +++ b/internal/kms/kms/gcp/gcp.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kms/kms/internal/internal.go b/internal/kms/kms/internal/internal.go index 914295a43..b6af19ef8 100644 --- a/internal/kms/kms/internal/internal.go +++ b/internal/kms/kms/internal/internal.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kms/kms/internal/internal_test.go b/internal/kms/kms/internal/internal_test.go index c6371a607..3058b4d7c 100644 --- a/internal/kms/kms/internal/internal_test.go +++ b/internal/kms/kms/internal/internal_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package internal @@ -27,6 +27,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -134,7 +135,7 @@ func TestGetDEK(t *testing.T) { Storage: tc.storage, } - dek, err := client.GetDEK(context.Background(), "volume-01", 32) + dek, err := client.GetDEK(t.Context(), "volume-01", 32) if tc.wantErr { assert.Error(err) } else { diff --git a/internal/kms/kms/kms.go b/internal/kms/kms/kms.go index d14eb435e..fe63957f2 100644 --- a/internal/kms/kms/kms.go +++ b/internal/kms/kms/kms.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package kms provides an abstract interface for Key Management Services. diff --git a/internal/kms/setup/setup.go b/internal/kms/setup/setup.go index eee089e77..99f4bcf6c 100644 --- a/internal/kms/setup/setup.go +++ b/internal/kms/setup/setup.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kms/setup/setup_test.go b/internal/kms/setup/setup_test.go index a92bbc0c1..1c8ee75e4 100644 --- a/internal/kms/setup/setup_test.go +++ b/internal/kms/setup/setup_test.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package setup import ( - "context" "testing" "github.com/edgelesssys/constellation/v2/internal/kms/uri" @@ -19,18 +18,19 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } func TestSetUpKMS(t *testing.T) { assert := assert.New(t) - kms, err := KMS(context.Background(), "storage://unknown", "kms://unknown") + kms, err := KMS(t.Context(), "storage://unknown", "kms://unknown") assert.Error(err) assert.Nil(kms) masterSecret := uri.MasterSecret{Key: []byte("key"), Salt: []byte("salt")} - kms, err = KMS(context.Background(), "storage://no-store", masterSecret.EncodeToURI()) + kms, err = KMS(t.Context(), "storage://no-store", masterSecret.EncodeToURI()) assert.NoError(err) assert.NotNil(kms) } diff --git a/internal/kms/storage/awss3/awss3.go b/internal/kms/storage/awss3/awss3.go index 535ab944e..c3d59c503 100644 --- a/internal/kms/storage/awss3/awss3.go +++ b/internal/kms/storage/awss3/awss3.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package awss3 implements a storage backend for the KMS using AWS S3: https://aws.amazon.com/s3/ diff --git a/internal/kms/storage/awss3/awss3_test.go b/internal/kms/storage/awss3/awss3_test.go index 4e07ab84d..153bbd209 100644 --- a/internal/kms/storage/awss3/awss3_test.go +++ b/internal/kms/storage/awss3/awss3_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package awss3 @@ -80,7 +80,7 @@ func TestAWSS3Get(t *testing.T) { client: tc.client, } - out, err := store.Get(context.Background(), "test-key") + out, err := store.Get(t.Context(), "test-key") if tc.wantErr { assert.Error(err) @@ -122,7 +122,7 @@ func TestAWSS3Put(t *testing.T) { testData := []byte{0x1, 0x2, 0x3} - err := store.Put(context.Background(), "test-key", testData) + err := store.Put(t.Context(), "test-key", testData) if tc.wantErr { assert.Error(err) } else { @@ -163,7 +163,7 @@ func TestAWSS3CreateBucket(t *testing.T) { client: tc.client, } - err := store.createBucket(context.Background(), "test-bucket", "test-region") + err := store.createBucket(t.Context(), "test-bucket", "test-region") if tc.wantErr { assert.Error(err) } else { diff --git a/internal/kms/storage/azureblob/azureblob.go b/internal/kms/storage/azureblob/azureblob.go index e7e41424e..36483a684 100644 --- a/internal/kms/storage/azureblob/azureblob.go +++ b/internal/kms/storage/azureblob/azureblob.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package azureblob implements a storage backend for the KMS using Azure Blob Storage. diff --git a/internal/kms/storage/azureblob/azureblob_test.go b/internal/kms/storage/azureblob/azureblob_test.go index 93a5f2987..19c590be4 100644 --- a/internal/kms/storage/azureblob/azureblob_test.go +++ b/internal/kms/storage/azureblob/azureblob_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package azureblob @@ -51,7 +51,7 @@ func TestAzureGet(t *testing.T) { container: "test", } - out, err := client.Get(context.Background(), "test-key") + out, err := client.Get(t.Context(), "test-key") if tc.wantErr { assert.Error(err) @@ -93,7 +93,7 @@ func TestAzurePut(t *testing.T) { container: "test", } - err := client.Put(context.Background(), "test-key", testData) + err := client.Put(t.Context(), "test-key", testData) if tc.wantErr { assert.Error(err) return @@ -130,7 +130,7 @@ func TestCreateContainerOrContinue(t *testing.T) { container: "test", } - err := client.createContainerOrContinue(context.Background()) + err := client.createContainerOrContinue(t.Context()) if tc.wantErr { assert.Error(err) } else { diff --git a/internal/kms/storage/gcs/gcs.go b/internal/kms/storage/gcs/gcs.go index ca53bf55f..f3c19ef2b 100644 --- a/internal/kms/storage/gcs/gcs.go +++ b/internal/kms/storage/gcs/gcs.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package gcs implements a storage backend for the KMS using Google Cloud Storage (GCS). diff --git a/internal/kms/storage/gcs/gcs_test.go b/internal/kms/storage/gcs/gcs_test.go index 5678afee5..7d3d8dd27 100644 --- a/internal/kms/storage/gcs/gcs_test.go +++ b/internal/kms/storage/gcs/gcs_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package gcs @@ -103,7 +103,7 @@ func TestGCPGet(t *testing.T) { bucketName: "test", } - out, err := client.Get(context.Background(), "test-key") + out, err := client.Get(t.Context(), "test-key") if tc.wantErr { assert.Error(err) @@ -160,7 +160,7 @@ func TestGCPPut(t *testing.T) { } testData := []byte{0x1, 0x2, 0x3} - err := client.Put(context.Background(), "test-key", testData) + err := client.Put(t.Context(), "test-key", testData) if tc.wantErr { assert.Error(err) } else { @@ -211,7 +211,7 @@ func TestGCPCreateContainerOrContinue(t *testing.T) { bucketName: "test", } - err := client.createContainerOrContinue(context.Background(), "project") + err := client.createContainerOrContinue(t.Context(), "project") if tc.wantErr { assert.Error(err) } else { diff --git a/internal/kms/storage/memfs/memfs.go b/internal/kms/storage/memfs/memfs.go index 3acb4ca53..98f2d65af 100644 --- a/internal/kms/storage/memfs/memfs.go +++ b/internal/kms/storage/memfs/memfs.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package memfs implements a storage backend for the KMS that stores keys in memory only. diff --git a/internal/kms/storage/memfs/memfs_test.go b/internal/kms/storage/memfs/memfs_test.go index 9fe33362b..cad508632 100644 --- a/internal/kms/storage/memfs/memfs_test.go +++ b/internal/kms/storage/memfs/memfs_test.go @@ -1,13 +1,12 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package memfs import ( - "context" "testing" "github.com/edgelesssys/constellation/v2/internal/kms/storage" @@ -19,6 +18,7 @@ func TestMain(m *testing.M) { goleak.VerifyTestMain(m, // https://github.com/census-instrumentation/opencensus-go/issues/1262 goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"), ) } @@ -29,7 +29,7 @@ func TestMemMapStorage(t *testing.T) { testDEK1 := []byte("test DEK") testDEK2 := []byte("more test DEK") - ctx := context.Background() + ctx := t.Context() // request unset value _, err := store.Get(ctx, "test:input") diff --git a/internal/kms/storage/storage.go b/internal/kms/storage/storage.go index d8ec42c1e..21cc04146 100644 --- a/internal/kms/storage/storage.go +++ b/internal/kms/storage/storage.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kms/test/aws_test.go b/internal/kms/test/aws_test.go index 184d045a0..bc1084c5b 100644 --- a/internal/kms/test/aws_test.go +++ b/internal/kms/test/aws_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package test @@ -34,7 +34,7 @@ func TestAwsStorage(t *testing.T) { } require := require.New(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() // create bucket @@ -65,7 +65,7 @@ func cleanUpBucket(ctx context.Context, require *require.Assertions, bucketID, a require.NoError(err) var objects []string var i int32 - for i = 0; i < output.KeyCount; i++ { + for i = 0; i < *output.KeyCount; i++ { objects = append(objects, *output.Contents[i].Key) } // Delete all objects of the bucket @@ -105,7 +105,7 @@ func TestAwsKms(t *testing.T) { require := require.New(t) store := memfs.New() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() cfg := uri.AWSConfig{ diff --git a/internal/kms/test/azure_test.go b/internal/kms/test/azure_test.go index 855b4dd54..d5633b70a 100644 --- a/internal/kms/test/azure_test.go +++ b/internal/kms/test/azure_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package test @@ -31,7 +31,7 @@ func TestAzureStorage(t *testing.T) { } require := require.New(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() cfg := uri.AzureBlobConfig{ @@ -59,7 +59,7 @@ func TestAzureKeyKMS(t *testing.T) { require := require.New(t) store := memfs.New() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() cfg := uri.AzureConfig{ @@ -88,7 +88,7 @@ func TestAzureKeyHSM(t *testing.T) { require := require.New(t) store := memfs.New() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() cfg := uri.AzureConfig{ diff --git a/internal/kms/test/gcp_test.go b/internal/kms/test/gcp_test.go index 35162e0f1..598db9c13 100644 --- a/internal/kms/test/gcp_test.go +++ b/internal/kms/test/gcp_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package test @@ -32,7 +32,7 @@ func TestGCPKMS(t *testing.T) { require := require.New(t) store := memfs.New() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() cfg := uri.GCPConfig{ @@ -59,7 +59,7 @@ func TestGcpStorage(t *testing.T) { } require := require.New(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() cfg := uri.GoogleCloudStorageConfig{ diff --git a/internal/kms/test/integration_test.go b/internal/kms/test/integration_test.go index bd6dccd80..d63834f44 100644 --- a/internal/kms/test/integration_test.go +++ b/internal/kms/test/integration_test.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package test provides integration tests for KMS and storage backends. @@ -64,7 +64,7 @@ func runKMSTest(t *testing.T, kms kms.CloudKMS) { dekName := "test-dek" - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() res, err := kms.GetDEK(ctx, dekName, config.SymmetricKeyLength) @@ -90,7 +90,7 @@ func runStorageTest(t *testing.T, store kms.Storage) { testData := []byte("Constellation test data") testName := "constellation-test" - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) defer cancel() err := store.Put(ctx, testName, testData) diff --git a/internal/kms/uri/uri.go b/internal/kms/uri/uri.go index 6a3de8887..bcc3a5d5e 100644 --- a/internal/kms/uri/uri.go +++ b/internal/kms/uri/uri.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kms/uri/uri_test.go b/internal/kms/uri/uri_test.go index 3517d2a8c..5532dc2c3 100644 --- a/internal/kms/uri/uri_test.go +++ b/internal/kms/uri/uri_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package uri @@ -15,7 +15,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestMasterSecretURI(t *testing.T) { diff --git a/internal/kubernetes/configmaps.go b/internal/kubernetes/configmaps.go index 0aed90a05..3ad5f90af 100644 --- a/internal/kubernetes/configmaps.go +++ b/internal/kubernetes/configmaps.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/internal/kubernetes/configmaps_test.go b/internal/kubernetes/configmaps_test.go index 96c3f475d..702ab4d5e 100644 --- a/internal/kubernetes/configmaps_test.go +++ b/internal/kubernetes/configmaps_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/internal/kubernetes/kubectl/kubectl.go b/internal/kubernetes/kubectl/kubectl.go index f61488082..2e8ddd7f5 100644 --- a/internal/kubernetes/kubectl/kubectl.go +++ b/internal/kubernetes/kubectl/kubectl.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* @@ -188,65 +188,6 @@ func (k *Kubectl) PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR st return err } -// EnforceCoreDNSSpread adds a pod anti-affinity to the CoreDNS deployment to ensure that -// CoreDNS pods are spread across nodes. -func (k *Kubectl) EnforceCoreDNSSpread(ctx context.Context) error { - // allow CoreDNS Pods to run on uninitialized nodes, which is required by cloud-controller-manager - tolerationSeconds := int64(10) - tolerations := []corev1.Toleration{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Value: "true", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "node.kubernetes.io/unreachable", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoExecute, - TolerationSeconds: &tolerationSeconds, - }, - } - - deployments := k.AppsV1().Deployments("kube-system") - // retry resource update if an error occurs - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - result, err := deployments.Get(ctx, "coredns", metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get Deployment to add toleration: %w", err) - } - - result.Spec.Template.Spec.Tolerations = append(result.Spec.Template.Spec.Tolerations, tolerations...) - - if result.Spec.Template.Spec.Affinity == nil { - result.Spec.Template.Spec.Affinity = &corev1.Affinity{} - } - if result.Spec.Template.Spec.Affinity.PodAntiAffinity == nil { - result.Spec.Template.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{} - } - result.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.WeightedPodAffinityTerm{} - if result.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - result.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []corev1.PodAffinityTerm{} - } - - result.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(result.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, - corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "k8s-app", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"kube-dns"}, - }, - }, - }, - TopologyKey: "kubernetes.io/hostname", - }) - - _, err = deployments.Update(ctx, result, metav1.UpdateOptions{}) - return err - }) -} - // AddNodeSelectorsToDeployment adds [K8s selectors] to the deployment, identified // by name and namespace. // diff --git a/internal/kubernetes/kubectl/kubectl_test.go b/internal/kubernetes/kubectl/kubectl_test.go index 3ca00e51d..5bdee84f7 100644 --- a/internal/kubernetes/kubectl/kubectl_test.go +++ b/internal/kubernetes/kubectl/kubectl_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubectl diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index cf8c478da..6d43c1b51 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* diff --git a/internal/kubernetes/marshal.go b/internal/kubernetes/marshal.go index d402ce824..958cbf956 100644 --- a/internal/kubernetes/marshal.go +++ b/internal/kubernetes/marshal.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/internal/kubernetes/marshal_test.go b/internal/kubernetes/marshal_test.go index 9da401000..2dff4d4fd 100644 --- a/internal/kubernetes/marshal_test.go +++ b/internal/kubernetes/marshal_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/internal/kubernetes/secrets.go b/internal/kubernetes/secrets.go index 4c8847c61..7cdfc848b 100644 --- a/internal/kubernetes/secrets.go +++ b/internal/kubernetes/secrets.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/internal/kubernetes/secrets_test.go b/internal/kubernetes/secrets_test.go index bc91da831..972d539bc 100644 --- a/internal/kubernetes/secrets_test.go +++ b/internal/kubernetes/secrets_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package kubernetes diff --git a/internal/license/BUILD.bazel b/internal/license/BUILD.bazel index 4cd4e56a8..58764259a 100644 --- a/internal/license/BUILD.bazel +++ b/internal/license/BUILD.bazel @@ -22,11 +22,11 @@ go_library( go_test( name = "license_test", - srcs = [ - "file_test.go", - "license_test.go", - ], + srcs = ["file_test.go"], embed = [":license"], + tags = [ + "enterprise", + ], deps = [ "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", diff --git a/internal/license/checker_enterprise.go b/internal/license/checker_enterprise.go index 91b6fd753..9807c992c 100644 --- a/internal/license/checker_enterprise.go +++ b/internal/license/checker_enterprise.go @@ -3,32 +3,96 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package license import ( + "bytes" "context" + "encoding/json" + "fmt" + "net/http" + "net/url" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" ) +const ( + apiHost = "license.confidential.cloud" + licensePath = "api/v1/license" +) + +// Checker checks the Constellation license. type Checker struct { - quotaChecker QuotaChecker + httpClient *http.Client } -func NewChecker(quotaChecker QuotaChecker) *Checker { +// NewChecker creates a new Checker. +func NewChecker() *Checker { return &Checker{ - quotaChecker: quotaChecker, + httpClient: http.DefaultClient, } } -// CheckLicense contacts the license server to fetch quota information for the given license. -func (c *Checker) CheckLicense(ctx context.Context, provider cloudprovider.Provider, licenseID string) (QuotaCheckResponse, error) { - return c.quotaChecker.QuotaCheck(ctx, QuotaCheckRequest{ +// CheckLicense checks the Constellation license. If the license is valid, it returns the vCPU quota. +func (c *Checker) CheckLicense(ctx context.Context, csp cloudprovider.Provider, action Action, licenseID string) (int, error) { + checkRequest := quotaCheckRequest{ + Provider: csp.String(), License: licenseID, - Action: Init, - Provider: provider.String(), - }) + Action: action, + } + + reqBody, err := json.Marshal(checkRequest) + if err != nil { + return 0, fmt.Errorf("unable to marshal input: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, licenseURL().String(), bytes.NewBuffer(reqBody)) + if err != nil { + return 0, fmt.Errorf("unable to create request: %w", err) + } + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, fmt.Errorf("unable to do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, fmt.Errorf("http error %d", resp.StatusCode) + } + + responseContentType := resp.Header.Get("Content-Type") + if responseContentType != "application/json" { + return 0, fmt.Errorf("expected server JSON response but got '%s'", responseContentType) + } + + var parsedResponse quotaCheckResponse + err = json.NewDecoder(resp.Body).Decode(&parsedResponse) + if err != nil { + return 0, fmt.Errorf("unable to parse response: %w", err) + } + + return parsedResponse.Quota, nil +} + +// quotaCheckRequest is JSON request to license server to check quota for a given license and action. +type quotaCheckRequest struct { + Action Action `json:"action"` + Provider string `json:"provider"` + License string `json:"license"` +} + +// quotaCheckResponse is JSON response by license server. +type quotaCheckResponse struct { + Quota int `json:"quota"` +} + +func licenseURL() *url.URL { + return &url.URL{ + Scheme: "https", + Host: apiHost, + Path: licensePath, + } } diff --git a/internal/license/license_test.go b/internal/license/checker_enterprise_test.go similarity index 66% rename from internal/license/license_test.go rename to internal/license/checker_enterprise_test.go index 61667581a..fd35b786c 100644 --- a/internal/license/license_test.go +++ b/internal/license/checker_enterprise_test.go @@ -1,18 +1,20 @@ +//go:build enterprise + /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package license import ( "bytes" - "context" "io" "net/http" "testing" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/stretchr/testify/assert" ) @@ -25,11 +27,9 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { } // newTestClient returns *http.Client with Transport replaced to avoid making real calls. -func newTestClient(fn roundTripFunc) *Client { - return &Client{ - httpClient: &http.Client{ - Transport: fn, - }, +func newTestClient(fn roundTripFunc) *http.Client { + return &http.Client{ + Transport: fn, } } @@ -70,31 +70,26 @@ func TestQuotaCheck(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - client := newTestClient(func(req *http.Request) *http.Response { - r := &http.Response{ - StatusCode: tc.serverResponseCode, - Body: io.NopCloser(bytes.NewBufferString(tc.serverResponse)), - Header: make(http.Header), - } - r.Header.Set("Content-Type", tc.serverResponseContent) - return r - }) + client := &Checker{ + httpClient: newTestClient(func(req *http.Request) *http.Response { + r := &http.Response{ + StatusCode: tc.serverResponseCode, + Body: io.NopCloser(bytes.NewBufferString(tc.serverResponse)), + Header: make(http.Header), + } + r.Header.Set("Content-Type", tc.serverResponseContent) + return r + }), + } - resp, err := client.QuotaCheck(context.Background(), QuotaCheckRequest{ - Action: test, - License: tc.license, - }) + quota, err := client.CheckLicense(t.Context(), cloudprovider.Unknown, Init, tc.license) if tc.wantError { assert.Error(err) return } assert.NoError(err) - assert.Equal(tc.wantQuota, resp.Quota) + assert.Equal(tc.wantQuota, quota) }) } } - -func Test_licenseURL(t *testing.T) { - assert.Equal(t, "https://license.confidential.cloud/api/v1/license", licenseURL().String()) -} diff --git a/internal/license/checker_oss.go b/internal/license/checker_oss.go index a7b18327c..3ada97f0a 100644 --- a/internal/license/checker_oss.go +++ b/internal/license/checker_oss.go @@ -3,7 +3,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package license @@ -18,11 +18,11 @@ import ( type Checker struct{} // NewChecker creates a new Checker. -func NewChecker(QuotaChecker) *Checker { +func NewChecker() *Checker { return &Checker{} } // CheckLicense is a no-op for open source version of Constellation. -func (c *Checker) CheckLicense(context.Context, cloudprovider.Provider, string) (QuotaCheckResponse, error) { - return QuotaCheckResponse{}, nil +func (c *Checker) CheckLicense(context.Context, cloudprovider.Provider, Action, string) (int, error) { + return 0, nil } diff --git a/internal/license/file.go b/internal/license/file.go index 01f5afdff..9df7d6ab0 100644 --- a/internal/license/file.go +++ b/internal/license/file.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package license diff --git a/internal/license/file_test.go b/internal/license/file_test.go index 84101dd72..3114a5f27 100644 --- a/internal/license/file_test.go +++ b/internal/license/file_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package license diff --git a/internal/license/integration/BUILD.bazel b/internal/license/integration/BUILD.bazel index fe71e702a..0a84478da 100644 --- a/internal/license/integration/BUILD.bazel +++ b/internal/license/integration/BUILD.bazel @@ -4,10 +4,12 @@ go_test( name = "integration_test", srcs = ["license_integration_test.go"], tags = [ + "enterprise", "integration", "requires-network", ], deps = [ + "//internal/cloud/cloudprovider", "//internal/license", "@com_github_stretchr_testify//assert", ], diff --git a/internal/license/integration/license_integration_test.go b/internal/license/integration/license_integration_test.go index 7a6158262..64ba47011 100644 --- a/internal/license/integration/license_integration_test.go +++ b/internal/license/integration/license_integration_test.go @@ -3,15 +3,15 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package integration import ( - "context" "testing" + "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/license" "github.com/stretchr/testify/assert" ) @@ -36,20 +36,16 @@ func TestQuotaCheckIntegration(t *testing.T) { t.Run(name, func(t *testing.T) { assert := assert.New(t) - client := license.NewClient() + client := license.NewChecker() - req := license.QuotaCheckRequest{ - Action: license.Action("test"), - License: tc.license, - } - resp, err := client.QuotaCheck(context.Background(), req) + quota, err := client.CheckLicense(t.Context(), cloudprovider.Unknown, "test", tc.license) if tc.wantError { assert.Error(err) return } assert.NoError(err) - assert.Equal(tc.wantQuota, resp.Quota) + assert.Equal(tc.wantQuota, quota) }) } } diff --git a/internal/license/license.go b/internal/license/license.go index 1e9525361..5fcd91f98 100644 --- a/internal/license/license.go +++ b/internal/license/license.go @@ -1,107 +1,24 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package license provides functions to check a user's Constellation license. package license -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "net/url" -) +// Action performed by Constellation. +type Action string const ( // CommunityLicense is used by everyone who has not bought an enterprise license. CommunityLicense = "00000000-0000-0000-0000-000000000000" - apiHost = "license.confidential.cloud" - licensePath = "api/v1/license" -) + // MarketplaceLicense is used by everyone who uses a marketplace image. + MarketplaceLicense = "11111111-1111-1111-1111-111111111111" -type ( - // Action performed by Constellation. - Action string -) - -const ( // Init action denotes the initialization of a Constellation cluster. Init Action = "init" - // test action is only to be used in testing. - test Action = "test" + // Apply action denotes an update of a Constellation cluster. + // It is used after a cluster has already been initialized once. + Apply Action = "apply" ) - -// Client interacts with the ES license server. -type Client struct { - httpClient *http.Client -} - -// NewClient creates a new client to interact with ES license server. -func NewClient() *Client { - return &Client{ - httpClient: http.DefaultClient, - } -} - -// QuotaCheckRequest is JSON request to license server to check quota for a given license and action. -type QuotaCheckRequest struct { - Action Action `json:"action"` - Provider string `json:"provider"` - License string `json:"license"` -} - -// QuotaCheckResponse is JSON response by license server. -type QuotaCheckResponse struct { - Quota int `json:"quota"` -} - -// QuotaCheck for a given license and action, passed via CheckQuotaRequest. -func (c *Client) QuotaCheck(ctx context.Context, checkRequest QuotaCheckRequest) (QuotaCheckResponse, error) { - reqBody, err := json.Marshal(checkRequest) - if err != nil { - return QuotaCheckResponse{}, fmt.Errorf("unable to marshal input: %w", err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, licenseURL().String(), bytes.NewBuffer(reqBody)) - if err != nil { - return QuotaCheckResponse{}, fmt.Errorf("unable to create request: %w", err) - } - resp, err := c.httpClient.Do(req) - if err != nil { - return QuotaCheckResponse{}, fmt.Errorf("unable to do request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return QuotaCheckResponse{}, fmt.Errorf("http error %d", resp.StatusCode) - } - - responseContentType := resp.Header.Get("Content-Type") - if responseContentType != "application/json" { - return QuotaCheckResponse{}, fmt.Errorf("expected server JSON response but got '%s'", responseContentType) - } - - var parsedResponse QuotaCheckResponse - err = json.NewDecoder(resp.Body).Decode(&parsedResponse) - if err != nil { - return QuotaCheckResponse{}, fmt.Errorf("unable to parse response: %w", err) - } - - return parsedResponse, nil -} - -func licenseURL() *url.URL { - return &url.URL{ - Scheme: "https", - Host: apiHost, - Path: licensePath, - } -} - -// QuotaChecker checks the vCPU quota for a given license. -type QuotaChecker interface { - QuotaCheck(ctx context.Context, checkRequest QuotaCheckRequest) (QuotaCheckResponse, error) -} diff --git a/internal/logger/BUILD.bazel b/internal/logger/BUILD.bazel index f1b95ba19..10f753f44 100644 --- a/internal/logger/BUILD.bazel +++ b/internal/logger/BUILD.bazel @@ -5,16 +5,14 @@ go_library( srcs = [ "cmdline.go", "grpclogger.go", + "levelhandler.go", "log.go", ], importpath = "github.com/edgelesssys/constellation/v2/internal/logger", visibility = ["//:__subpackages__"], deps = [ "@com_github_grpc_ecosystem_go_grpc_middleware_v2//interceptors/logging", - "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//grpclog", - "@org_uber_go_zap//:zap", - "@org_uber_go_zap//zapcore", - "@org_uber_go_zap//zaptest", ], ) diff --git a/internal/logger/cmdline.go b/internal/logger/cmdline.go index 4957e05ca..9bdfa95cf 100644 --- a/internal/logger/cmdline.go +++ b/internal/logger/cmdline.go @@ -1,31 +1,30 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package logger import ( - "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "log/slog" ) // CmdLineVerbosityDescription explains numeric log levels. -const CmdLineVerbosityDescription = "log verbosity in zap logging levels. Use -1 for debug information, 0 for info, 1 for warn, 2 for error" +const CmdLineVerbosityDescription = "log verbosity: Use -1 for debug information, 0 for info, 1 for warn, 2 for error" -// VerbosityFromInt converts a verbosity level from an integer to a zapcore.Level. -func VerbosityFromInt(verbosity int) zapcore.Level { +// VerbosityFromInt converts a verbosity level from an integer to a slog.Level. +func VerbosityFromInt(verbosity int) slog.Level { switch { case verbosity <= -1: - return zap.DebugLevel + return slog.LevelDebug case verbosity == 0: - return zap.InfoLevel + return slog.LevelInfo case verbosity == 1: - return zap.WarnLevel + return slog.LevelWarn case verbosity >= 2: - return zap.ErrorLevel + return slog.LevelError default: - return zap.InfoLevel + return slog.LevelInfo } } diff --git a/internal/logger/grpclogger.go b/internal/logger/grpclogger.go index 3381df30a..fead5cf8a 100644 --- a/internal/logger/grpclogger.go +++ b/internal/logger/grpclogger.go @@ -1,77 +1,102 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package logger import ( + "context" "fmt" + "log/slog" + "os" + "runtime" + "time" - "go.uber.org/zap" "google.golang.org/grpc/grpclog" ) -func replaceGRPCLogger(log *zap.Logger) { +func replaceGRPCLogger(log *slog.Logger) { gl := &grpcLogger{ - logger: log.With(zap.String("system", "grpc"), zap.Bool("grpc_log", true)).WithOptions(zap.AddCallerSkip(2)), + logger: log, verbosity: 0, } grpclog.SetLoggerV2(gl) } +func (l *grpcLogger) log(level slog.Level, args ...interface{}) { + if l.logger.Enabled(context.Background(), level) { + var pcs [1]uintptr + runtime.Callers(3, pcs[:]) + r := slog.NewRecord(time.Now(), level, fmt.Sprint(args...), pcs[0]) + _ = l.logger.Handler().Handle(context.Background(), r) + } +} + +func (l *grpcLogger) logf(level slog.Level, format string, args ...interface{}) { + if l.logger.Enabled(context.Background(), level) { + var pcs [1]uintptr + runtime.Callers(3, pcs[:]) + r := slog.NewRecord(time.Now(), level, fmt.Sprintf(format, args...), pcs[0]) + _ = l.logger.Handler().Handle(context.Background(), r) + } +} + type grpcLogger struct { - logger *zap.Logger + logger *slog.Logger verbosity int } func (l *grpcLogger) Info(args ...interface{}) { - l.logger.Info(fmt.Sprint(args...)) + l.log(slog.LevelInfo, args...) } func (l *grpcLogger) Infoln(args ...interface{}) { - l.logger.Info(fmt.Sprint(args...)) + l.log(slog.LevelInfo, args...) } func (l *grpcLogger) Infof(format string, args ...interface{}) { - l.logger.Info(fmt.Sprintf(format, args...)) + l.logf(slog.LevelInfo, format, args...) } func (l *grpcLogger) Warning(args ...interface{}) { - l.logger.Warn(fmt.Sprint(args...)) + l.log(slog.LevelWarn, args...) } func (l *grpcLogger) Warningln(args ...interface{}) { - l.logger.Warn(fmt.Sprint(args...)) + l.log(slog.LevelWarn, args...) } func (l *grpcLogger) Warningf(format string, args ...interface{}) { - l.logger.Warn(fmt.Sprintf(format, args...)) + l.logf(slog.LevelWarn, format, args...) } func (l *grpcLogger) Error(args ...interface{}) { - l.logger.Error(fmt.Sprint(args...)) + l.log(slog.LevelError, args...) } func (l *grpcLogger) Errorln(args ...interface{}) { - l.logger.Error(fmt.Sprint(args...)) + l.log(slog.LevelError, args...) } func (l *grpcLogger) Errorf(format string, args ...interface{}) { - l.logger.Error(fmt.Sprintf(format, args...)) + l.logf(slog.LevelError, format, args...) } func (l *grpcLogger) Fatal(args ...interface{}) { - l.logger.Fatal(fmt.Sprint(args...)) + l.log(slog.LevelError, args...) + os.Exit(1) } func (l *grpcLogger) Fatalln(args ...interface{}) { - l.logger.Fatal(fmt.Sprint(args...)) + l.log(slog.LevelError, args...) + os.Exit(1) } func (l *grpcLogger) Fatalf(format string, args ...interface{}) { - l.logger.Fatal(fmt.Sprintf(format, args...)) + l.logf(slog.LevelError, format, args...) + os.Exit(1) } func (l *grpcLogger) V(level int) bool { diff --git a/internal/logger/levelhandler.go b/internal/logger/levelhandler.go new file mode 100644 index 000000000..d9b4cec29 --- /dev/null +++ b/internal/logger/levelhandler.go @@ -0,0 +1,57 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package logger + +import ( + "context" + "log/slog" +) + +// LevelHandler copied from the official LevelHandler example in the slog package documentation. + +// levelHandler wraps a Handler with an Enabled method +// that returns false for levels below a minimum. +type levelHandler struct { + level slog.Leveler + handler slog.Handler +} + +// newLevelHandler returns a LevelHandler with the given level. +// All methods except Enabled delegate to h. +func newLevelHandler(level slog.Leveler, h slog.Handler) *levelHandler { + // Optimization: avoid chains of LevelHandlers. + if lh, ok := h.(*levelHandler); ok { + h = lh.Handler() + } + return &levelHandler{level, h} +} + +// Enabled implements Handler.Enabled by reporting whether +// level is at least as large as h's level. +func (h *levelHandler) Enabled(_ context.Context, level slog.Level) bool { + return level >= h.level.Level() +} + +// Handle implements Handler.Handle. +func (h *levelHandler) Handle(ctx context.Context, r slog.Record) error { + return h.handler.Handle(ctx, r) +} + +// WithAttrs implements Handler.WithAttrs. +func (h *levelHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return newLevelHandler(h.level, h.handler.WithAttrs(attrs)) +} + +// WithGroup implements Handler.WithGroup. +func (h *levelHandler) WithGroup(name string) slog.Handler { + return newLevelHandler(h.level, h.handler.WithGroup(name)) +} + +// Handler returns the Handler wrapped by h. +func (h *levelHandler) Handler() slog.Handler { + return h.handler +} diff --git a/internal/logger/log.go b/internal/logger/log.go index 55a9cb9c1..0f1b23789 100644 --- a/internal/logger/log.go +++ b/internal/logger/log.go @@ -1,227 +1,127 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ /* -Package logger provides logging functionality for Constellation services. -It is a thin wrapper around the zap package, providing a consistent interface for logging. -Use this package to implement logging for your Constellation services. +Package logger provides helper functions that can be used in combination with slog to increase functionality or make +working with slog easier. -# Usage +1. Logging in unit tests -1. Create a logger using New(). +To log in unit tests you can create a new slog logger that uses logger.testWriter as its writer. This can be constructed +by creating a logger like this: `logger.NewTest(t)`. -2. Defer the Sync() method to ensure that all log entries are flushed. +2. Creating a new logger with an increased log level based on another logger -3. Use the Debugf(), Infof(), Warnf(), Errorf(), and Fatalf() methods depending on the level of logging you need. +You can create a new logger with a new log level by creating a new slog.Logger with the LevelHandler in this package +and passing the handler of the other logger. As an example, if you have a slog.Logger named `log` you can create a +new logger with an increased log level (here slog.LevelWarn) like this: -4. Use the Named() method to create a named child logger. - -5. Use the With() method to create a child logger with structured context. -This can also be used to add context to a single log message: - - logger.With(zap.String("key", "value")).Infof("log message") - -# Log Levels - -Use [Logger.Debugf] to log low level and detailed information that is useful for debugging. - -Use [Logger.Infof] to log general information. This method is correct for most logging purposes. - -Use [Logger.Warnf] to log information that may indicate unwanted behavior, but is not an error. - -Use [Logger.Errorf] to log information about any errors that occurred. - -Use [Logger.Fatalf] to log information about any errors that occurred and then exit the program. + slog.New(logger.NewLevelHandler(slog.LevelWarn, log.Handler())) */ package logger import ( "context" - "fmt" + "log/slog" "os" + "runtime" "testing" + "time" "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "go.uber.org/zap/zaptest" "google.golang.org/grpc" ) -// LogType indicates the output encoding of the log. -type LogType int - -const ( - // JSONLog encodes logs in JSON format. - JSONLog LogType = iota - // PlainLog encodes logs as human readable text. - PlainLog -) - -// Logger is a wrapper for zap logger. -// The purpose is to provide a simple interface for logging with sensible defaults. -type Logger struct { - logger *zap.SugaredLogger -} - -// New creates a new Logger. -// Set name to an empty string to create an unnamed logger. -func New(logType LogType, logLevel zapcore.Level) *Logger { - encoderCfg := zap.NewProductionEncoderConfig() - encoderCfg.StacktraceKey = zapcore.OmitKey - encoderCfg.EncodeLevel = zapcore.CapitalLevelEncoder - encoderCfg.EncodeTime = zapcore.RFC3339TimeEncoder - - var encoder zapcore.Encoder - if logType == PlainLog { - encoder = zapcore.NewConsoleEncoder(encoderCfg) - } else { - encoder = zapcore.NewJSONEncoder(encoderCfg) - } - - logCore := zapcore.NewCore(encoder, zapcore.Lock(os.Stderr), zap.NewAtomicLevelAt(logLevel)) - - logger := zap.New( - logCore, - zap.AddCaller(), // add the file and line number of the logging call - zap.AddCallerSkip(1), // skip the first caller so that we don't only see this package as the caller - ) - - return &Logger{logger: logger.Sugar()} -} - -// NewTest creates a logger for unit / integration tests. -func NewTest(t *testing.T) *Logger { - return &Logger{ - logger: zaptest.NewLogger(t).Sugar().Named(fmt.Sprintf("%q", t.Name())), - } -} - -// Debugf logs a message at Debug level. -// Debug logs are typically voluminous, and contain detailed information on the flow of execution. -func (l *Logger) Debugf(format string, args ...any) { - l.logger.Debugf(format, args...) -} - -// Infof logs a message at Info level. -// This is the default logging priority and should be used for all normal messages. -func (l *Logger) Infof(format string, args ...any) { - l.logger.Infof(format, args...) -} - -// Warnf logs a message at Warn level. -// Warn logs are more important than Info, but they don't need human review or necessarily indicate an error. -func (l *Logger) Warnf(format string, args ...any) { - l.logger.Warnf(format, args...) -} - -// Errorf logs a message at Error level. -// Error logs are high priority and indicate something has gone wrong. -func (l *Logger) Errorf(format string, args ...any) { - l.logger.Errorf(format, args...) -} - -// Fatalf logs the message and then calls os.Exit(1). -// Use this to exit your program when a fatal error occurs. -func (l *Logger) Fatalf(format string, args ...any) { - l.logger.Fatalf(format, args...) -} - -// Sync flushes any buffered log entries. -// Applications should take care to call Sync before exiting. -func (l *Logger) Sync() { - _ = l.logger.Sync() -} - -// WithIncreasedLevel returns a logger with increased logging level. -func (l *Logger) WithIncreasedLevel(level zapcore.Level) *Logger { - return &Logger{logger: l.getZapLogger().WithOptions(zap.IncreaseLevel(level)).Sugar()} -} - -// With returns a logger with structured context. -func (l *Logger) With(fields ...any) *Logger { - return &Logger{logger: l.logger.With(fields...)} -} - -// Named returns a named logger. -func (l *Logger) Named(name string) *Logger { - return &Logger{logger: l.logger.Named(name)} +// GRPCLogger returns a logger at warn level for gRPC logging. +func GRPCLogger(l *slog.Logger) *slog.Logger { + return slog.New(newLevelHandler(slog.LevelWarn, l.Handler())).WithGroup("gRPC") } // ReplaceGRPCLogger replaces grpc's internal logger with the given logger. -func (l *Logger) ReplaceGRPCLogger() { - replaceGRPCLogger(l.getZapLogger()) +func ReplaceGRPCLogger(l *slog.Logger) { + replaceGRPCLogger(l) } // GetServerUnaryInterceptor returns a gRPC server option for intercepting unary gRPC logs. -func (l *Logger) GetServerUnaryInterceptor() grpc.ServerOption { +func GetServerUnaryInterceptor(l *slog.Logger) grpc.ServerOption { return grpc.UnaryInterceptor( - logging.UnaryServerInterceptor(l.middlewareLogger()), + logging.UnaryServerInterceptor(middlewareLogger(l)), ) } // GetServerStreamInterceptor returns a gRPC server option for intercepting streaming gRPC logs. -func (l *Logger) GetServerStreamInterceptor() grpc.ServerOption { +func GetServerStreamInterceptor(l *slog.Logger) grpc.ServerOption { return grpc.StreamInterceptor( - logging.StreamServerInterceptor(l.middlewareLogger()), + logging.StreamServerInterceptor(middlewareLogger(l)), ) } // GetClientUnaryInterceptor returns a gRPC client option for intercepting unary gRPC logs. -func (l *Logger) GetClientUnaryInterceptor() grpc.DialOption { +func GetClientUnaryInterceptor(l *slog.Logger) grpc.DialOption { return grpc.WithUnaryInterceptor( - logging.UnaryClientInterceptor(l.middlewareLogger()), + logging.UnaryClientInterceptor(middlewareLogger(l)), ) } // GetClientStreamInterceptor returns a gRPC client option for intercepting stream gRPC logs. -func (l *Logger) GetClientStreamInterceptor() grpc.DialOption { +func GetClientStreamInterceptor(l *slog.Logger) grpc.DialOption { return grpc.WithStreamInterceptor( - logging.StreamClientInterceptor(l.middlewareLogger()), + logging.StreamClientInterceptor(middlewareLogger(l)), ) } -// getZapLogger returns the underlying zap logger. -func (l *Logger) getZapLogger() *zap.Logger { - return l.logger.Desugar() -} - -func (l *Logger) middlewareLogger() logging.Logger { - return logging.LoggerFunc(func(ctx context.Context, lvl logging.Level, msg string, fields ...any) { - f := make([]zap.Field, 0, len(fields)/2) - - for i := 0; i < len(fields); i += 2 { - key := fields[i] - value := fields[i+1] - - switch v := value.(type) { - case string: - f = append(f, zap.String(key.(string), v)) - case int: - f = append(f, zap.Int(key.(string), v)) - case bool: - f = append(f, zap.Bool(key.(string), v)) - default: - f = append(f, zap.Any(key.(string), v)) - } - } - - logger := l.getZapLogger().WithOptions(zap.AddCallerSkip(1)).With(f...) +func middlewareLogger(l *slog.Logger) logging.Logger { + return logging.LoggerFunc(func(_ context.Context, lvl logging.Level, msg string, fields ...any) { + var pcs [1]uintptr + runtime.Callers(2, pcs[:]) // skip [Callers, LoggerFunc] + level := slog.LevelDebug switch lvl { case logging.LevelDebug: - logger.Debug(msg) + break case logging.LevelInfo: - logger.Info(msg) + level = slog.LevelInfo case logging.LevelWarn: - logger.Warn(msg) + level = slog.LevelWarn case logging.LevelError: - logger.Error(msg) + level = slog.LevelError default: - panic(fmt.Sprintf("unknown level %v", lvl)) + level = slog.LevelError } + + r := slog.NewRecord(time.Now(), level, msg, pcs[0]) + r.Add(fields...) + _ = l.Handler().Handle(context.Background(), r) }) } + +// NewTextLogger creates a new slog.Logger that writes text formatted log messages +// to os.Stderr. +func NewTextLogger(level slog.Level) *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{AddSource: true, Level: level})) +} + +// NewJSONLogger creates a new slog.Logger that writes JSON formatted log messages +// to os.Stderr. +func NewJSONLogger(level slog.Level) *slog.Logger { + return slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{AddSource: true, Level: level})) +} + +// NewTest creates a new slog.Logger that writes to a testing.T. +func NewTest(t *testing.T) *slog.Logger { + return slog.New(slog.NewTextHandler(testWriter{t: t}, &slog.HandlerOptions{AddSource: true})) +} + +// TestWriter is a writer to a testing.T used in tests for logging with slog. +type testWriter struct { + t *testing.T +} + +func (t testWriter) Write(p []byte) (int, error) { + t.t.Helper() + t.t.Log(string(p)) + return len(p), nil +} diff --git a/internal/maa/maa.go b/internal/maa/maa.go index fcbea6db7..cd1012cd7 100644 --- a/internal/maa/maa.go +++ b/internal/maa/maa.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package maa provides an interface for interacting with an MAA service diff --git a/internal/maa/patch.go b/internal/maa/patch.go index 5dfed9435..28b496658 100644 --- a/internal/maa/patch.go +++ b/internal/maa/patch.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package maa @@ -9,6 +9,7 @@ import ( "context" "encoding/base64" "fmt" + "io" "net/http" "github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation" @@ -55,10 +56,11 @@ func (p AzurePolicyPatcher) Patch(ctx context.Context, attestationURL string) er if err != nil { return fmt.Errorf("sending request: %w", err) } - resp.Body.Close() + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status) + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("updating attestation policy: unexpected status code: %s: %s", resp.Status, string(body)) } return nil diff --git a/internal/maa/patch_test.go b/internal/maa/patch_test.go index f00c30c7c..af87a8432 100644 --- a/internal/maa/patch_test.go +++ b/internal/maa/patch_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package maa diff --git a/internal/mpimage/mpimage.go b/internal/mpimage/mpimage.go index 89b6d1fa9..b25526d20 100644 --- a/internal/mpimage/mpimage.go +++ b/internal/mpimage/mpimage.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // The mpimage package provides utilities for handling CSP marketplace OS images. diff --git a/internal/mpimage/uri.go b/internal/mpimage/uri.go index 36c13afb2..9a41fafac 100644 --- a/internal/mpimage/uri.go +++ b/internal/mpimage/uri.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package mpimage diff --git a/internal/mpimage/uri_test.go b/internal/mpimage/uri_test.go index f7dfd3fe1..cf7eac912 100644 --- a/internal/mpimage/uri_test.go +++ b/internal/mpimage/uri_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package mpimage diff --git a/internal/nodestate/nodestate.go b/internal/nodestate/nodestate.go index 40e8113c7..e31dee7e8 100644 --- a/internal/nodestate/nodestate.go +++ b/internal/nodestate/nodestate.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // Package nodestate is used to persist the state of a Constellation node to disk. diff --git a/internal/nodestate/nodestate_test.go b/internal/nodestate/nodestate_test.go index 6b457eac9..576242a48 100644 --- a/internal/nodestate/nodestate_test.go +++ b/internal/nodestate/nodestate_test.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ package nodestate @@ -19,7 +19,7 @@ import ( ) func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + goleak.VerifyTestMain(m, goleak.IgnoreAnyFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1")) } func TestFromFile(t *testing.T) { diff --git a/internal/osimage/BUILD.bazel b/internal/osimage/BUILD.bazel index 3e9285662..0bf370060 100644 --- a/internal/osimage/BUILD.bazel +++ b/internal/osimage/BUILD.bazel @@ -8,6 +8,5 @@ go_library( deps = [ "//internal/api/versionsapi", "//internal/cloud/cloudprovider", - "//internal/osimage/secureboot", ], ) diff --git a/internal/osimage/archive/BUILD.bazel b/internal/osimage/archive/BUILD.bazel index 122ceef0c..4f90c37e4 100644 --- a/internal/osimage/archive/BUILD.bazel +++ b/internal/osimage/archive/BUILD.bazel @@ -8,7 +8,6 @@ go_library( deps = [ "//internal/api/versionsapi", "//internal/constants", - "//internal/logger", "//internal/staticupload", "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager", "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", diff --git a/internal/osimage/archive/archive.go b/internal/osimage/archive/archive.go index b31f20202..e4c9f6e2d 100644 --- a/internal/osimage/archive/archive.go +++ b/internal/osimage/archive/archive.go @@ -1,7 +1,7 @@ /* Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only +SPDX-License-Identifier: BUSL-1.1 */ // package archive is used to archive OS images in S3. @@ -9,7 +9,9 @@ package archive import ( "context" + "fmt" "io" + "log/slog" "net/url" "time" @@ -18,7 +20,6 @@ import ( s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/logger" "github.com/edgelesssys/constellation/v2/internal/staticupload" ) @@ -29,11 +30,11 @@ type Archivist struct { // bucket is the name of the S3 bucket to use. bucket string - log *logger.Logger + log *slog.Logger } // New creates a new Archivist. -func New(ctx context.Context, region, bucket, distributionID string, log *logger.Logger) (*Archivist, CloseFunc, error) { +func New(ctx context.Context, region, bucket, distributionID string, log *slog.Logger) (*Archivist, CloseFunc, error) { staticUploadClient, staticUploadClientClose, err := staticupload.New(ctx, staticupload.Config{ Region: region, Bucket: bucket, @@ -73,7 +74,7 @@ func (a *Archivist) Archive(ctx context.Context, version versionsapi.Version, cs if err != nil { return "", err } - a.log.Debugf("Archiving OS image %s %s %v to s3://%v/%v", csp, attestationVariant, version.ShortPath(), a.bucket, key) + a.log.Debug(fmt.Sprintf("Archiving OS image %q to s3://%s/%s", fmt.Sprintf("%s %s %v", csp, attestationVariant, version.ShortPath()), a.bucket, key)) _, err = a.uploadClient.Upload(ctx, &s3.PutObjectInput{ Bucket: &a.bucket, Key: &key, diff --git a/internal/osimage/aws/BUILD.bazel b/internal/osimage/aws/BUILD.bazel deleted file mode 100644 index 640714442..000000000 --- a/internal/osimage/aws/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "aws", - srcs = ["awsupload.go"], - importpath = "github.com/edgelesssys/constellation/v2/internal/osimage/aws", - visibility = ["//:__subpackages__"], - deps = [ - "//internal/api/versionsapi", - "//internal/logger", - "//internal/osimage", - "//internal/osimage/secureboot", - "@com_github_aws_aws_sdk_go_v2_config//:config", - "@com_github_aws_aws_sdk_go_v2_feature_s3_manager//:manager", - "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2", - "@com_github_aws_aws_sdk_go_v2_service_ec2//types", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", - "@com_github_aws_aws_sdk_go_v2_service_s3//types", - "@com_github_aws_smithy_go//:smithy-go", - ], -) diff --git a/internal/osimage/aws/awsupload.go b/internal/osimage/aws/awsupload.go deleted file mode 100644 index d5d097f47..000000000 --- a/internal/osimage/aws/awsupload.go +++ /dev/null @@ -1,603 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -// package aws implements uploading os images to aws. -package aws - -import ( - "context" - "errors" - "fmt" - "io" - "time" - - awsconfig "github.com/aws/aws-sdk-go-v2/config" - s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/ec2" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - - "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/osimage" - "github.com/edgelesssys/constellation/v2/internal/osimage/secureboot" -) - -// Uploader can upload and remove os images on GCP. -type Uploader struct { - region string - bucketName string - ec2 func(ctx context.Context, region string) (ec2API, error) - s3 func(ctx context.Context, region string) (s3API, error) - s3uploader func(ctx context.Context, region string) (s3UploaderAPI, error) - - log *logger.Logger -} - -// New creates a new Uploader. -func New(region, bucketName string, log *logger.Logger) (*Uploader, error) { - return &Uploader{ - region: region, - bucketName: bucketName, - ec2: func(ctx context.Context, region string) (ec2API, error) { - cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) - if err != nil { - return nil, err - } - return ec2.NewFromConfig(cfg), nil - }, - s3: func(ctx context.Context, region string) (s3API, error) { - cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) - if err != nil { - return nil, err - } - return s3.NewFromConfig(cfg), nil - }, - s3uploader: func(ctx context.Context, region string) (s3UploaderAPI, error) { - cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) - if err != nil { - return nil, err - } - return s3manager.NewUploader(s3.NewFromConfig(cfg)), nil - }, - - log: log, - }, nil -} - -// Upload uploads an OS image to AWS. -func (u *Uploader) Upload(ctx context.Context, req *osimage.UploadRequest) ([]versionsapi.ImageInfoEntry, error) { - blobName := fmt.Sprintf("image-%s-%s-%d.raw", req.Version.Stream(), req.Version.Version(), req.Timestamp.Unix()) - imageName := imageName(req.Version, req.AttestationVariant, req.Timestamp) - allRegions := []string{u.region} - allRegions = append(allRegions, replicationRegions...) - // TODO(malt3): make this configurable - publish := true - amiIDs := make(map[string]string, len(allRegions)) - if err := u.ensureBucket(ctx); err != nil { - return nil, fmt.Errorf("ensuring bucket %s exists: %w", u.bucketName, err) - } - - // pre-cleaning - for _, region := range allRegions { - if err := u.ensureImageDeleted(ctx, imageName, region); err != nil { - return nil, fmt.Errorf("pre-cleaning: ensuring no image under the name %s in region %s: %w", imageName, region, err) - } - } - if err := u.ensureSnapshotDeleted(ctx, imageName, u.region); err != nil { - return nil, fmt.Errorf("pre-cleaning: ensuring no snapshot using the same name exists: %w", err) - } - if err := u.ensureBlobDeleted(ctx, blobName); err != nil { - return nil, fmt.Errorf("pre-cleaning: ensuring no blob using the same name exists: %w", err) - } - - // create primary image - if err := u.uploadBlob(ctx, blobName, req.Image); err != nil { - return nil, fmt.Errorf("uploading image to s3: %w", err) - } - defer func() { - if err := u.ensureBlobDeleted(ctx, blobName); err != nil { - u.log.Errorf("post-cleaning: deleting temporary blob from s3", err) - } - }() - snapshotID, err := u.importSnapshot(ctx, blobName, imageName) - if err != nil { - return nil, fmt.Errorf("importing snapshot: %w", err) - } - primaryAMIID, err := u.createImageFromSnapshot(ctx, req.Version, imageName, snapshotID, req.SecureBoot, req.UEFIVarStore) - if err != nil { - return nil, fmt.Errorf("creating image from snapshot: %w", err) - } - amiIDs[u.region] = primaryAMIID - if err := u.waitForImage(ctx, primaryAMIID, u.region); err != nil { - return nil, fmt.Errorf("waiting for primary image to become available: %w", err) - } - - // replicate image - for _, region := range replicationRegions { - amiID, err := u.replicateImage(ctx, imageName, primaryAMIID, region) - if err != nil { - return nil, fmt.Errorf("replicating image to region %s: %w", region, err) - } - amiIDs[region] = amiID - } - - // wait for replication, tag, publish - var imageInfo []versionsapi.ImageInfoEntry - for _, region := range allRegions { - if err := u.waitForImage(ctx, amiIDs[region], region); err != nil { - return nil, fmt.Errorf("waiting for image to become available in region %s: %w", region, err) - } - if err := u.tagImageAndSnapshot(ctx, imageName, amiIDs[region], region); err != nil { - return nil, fmt.Errorf("tagging image in region %s: %w", region, err) - } - if !publish { - continue - } - if err := u.publishImage(ctx, amiIDs[region], region); err != nil { - return nil, fmt.Errorf("publishing image in region %s: %w", region, err) - } - imageInfo = append(imageInfo, versionsapi.ImageInfoEntry{ - CSP: "aws", - AttestationVariant: req.AttestationVariant, - Reference: amiIDs[region], - Region: region, - }) - } - - return imageInfo, nil -} - -func (u *Uploader) ensureBucket(ctx context.Context) error { - s3C, err := u.s3(ctx, u.region) - if err != nil { - return fmt.Errorf("determining if bucket %s exists: %w", u.bucketName, err) - } - _, err = s3C.HeadBucket(ctx, &s3.HeadBucketInput{ - Bucket: &u.bucketName, - }) - if err == nil { - u.log.Debugf("Bucket %s exists", u.bucketName) - return nil - } - var noSuchBucketErr *types.NoSuchBucket - if !errors.As(err, &noSuchBucketErr) { - return fmt.Errorf("determining if bucket %s exists: %w", u.bucketName, err) - } - u.log.Debugf("Creating bucket %s", u.bucketName) - _, err = s3C.CreateBucket(ctx, &s3.CreateBucketInput{ - Bucket: &u.bucketName, - }) - if err != nil { - return fmt.Errorf("creating bucket %s: %w", u.bucketName, err) - } - return nil -} - -func (u *Uploader) uploadBlob(ctx context.Context, blobName string, img io.Reader) error { - u.log.Debugf("Uploading os image as %s", blobName) - uploadC, err := u.s3uploader(ctx, u.region) - if err != nil { - return err - } - _, err = uploadC.Upload(ctx, &s3.PutObjectInput{ - Bucket: &u.bucketName, - Key: &blobName, - Body: img, - ChecksumAlgorithm: s3types.ChecksumAlgorithmSha256, - }) - return err -} - -func (u *Uploader) ensureBlobDeleted(ctx context.Context, blobName string) error { - s3C, err := u.s3(ctx, u.region) - if err != nil { - return err - } - _, err = s3C.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: &u.bucketName, - Key: &blobName, - }) - var apiError smithy.APIError - if errors.As(err, &apiError) && apiError.ErrorCode() == "NotFound" { - u.log.Debugf("Blob %s in %s doesn't exist. Nothing to clean up.", blobName, u.bucketName) - return nil - } - if err != nil { - return err - } - u.log.Debugf("Deleting blob %s", blobName) - _, err = s3C.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: &u.bucketName, - Key: &blobName, - }) - return err -} - -func (u *Uploader) findSnapshots(ctx context.Context, snapshotName, region string) ([]string, error) { - ec2C, err := u.ec2(ctx, region) - if err != nil { - return nil, fmt.Errorf("creating ec2 client: %w", err) - } - snapshots, err := ec2C.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{ - Filters: []ec2types.Filter{ - { - Name: toPtr("tag:Name"), - Values: []string{snapshotName}, - }, - }, - }) - if err != nil { - return nil, fmt.Errorf("describing snapshots: %w", err) - } - var snapshotIDs []string - for _, s := range snapshots.Snapshots { - if s.SnapshotId == nil { - continue - } - snapshotIDs = append(snapshotIDs, *s.SnapshotId) - } - return snapshotIDs, nil -} - -func (u *Uploader) importSnapshot(ctx context.Context, blobName, snapshotName string) (string, error) { - u.log.Debugf("Importing %s as snapshot %s", blobName, snapshotName) - ec2C, err := u.ec2(ctx, u.region) - if err != nil { - return "", fmt.Errorf("creating ec2 client: %w", err) - } - importResp, err := ec2C.ImportSnapshot(ctx, &ec2.ImportSnapshotInput{ - ClientData: &ec2types.ClientData{ - Comment: &snapshotName, - }, - Description: &snapshotName, - DiskContainer: &ec2types.SnapshotDiskContainer{ - Description: &snapshotName, - Format: toPtr(string(ec2types.DiskImageFormatRaw)), - UserBucket: &ec2types.UserBucket{ - S3Bucket: &u.bucketName, - S3Key: &blobName, - }, - }, - }) - if err != nil { - return "", fmt.Errorf("importing snapshot: %w", err) - } - if importResp.ImportTaskId == nil { - return "", fmt.Errorf("importing snapshot: no import task ID returned") - } - u.log.Debugf("Waiting for snapshot %s to be ready", snapshotName) - return waitForSnapshotImport(ctx, ec2C, *importResp.ImportTaskId) -} - -func (u *Uploader) ensureSnapshotDeleted(ctx context.Context, snapshotName, region string) error { - ec2C, err := u.ec2(ctx, region) - if err != nil { - return fmt.Errorf("creating ec2 client: %w", err) - } - snapshots, err := u.findSnapshots(ctx, snapshotName, region) - if err != nil { - return fmt.Errorf("finding snapshots: %w", err) - } - for _, snapshot := range snapshots { - u.log.Debugf("Deleting snapshot %s in %s", snapshot, region) - _, err = ec2C.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{ - SnapshotId: toPtr(snapshot), - }) - if err != nil { - return fmt.Errorf("deleting snapshot %s: %w", snapshot, err) - } - } - return nil -} - -func (u *Uploader) createImageFromSnapshot(ctx context.Context, version versionsapi.Version, imageName, snapshotID string, enableSecureBoot bool, uefiVarStore secureboot.UEFIVarStore) (string, error) { - u.log.Debugf("Creating image %s in %s", imageName, u.region) - ec2C, err := u.ec2(ctx, u.region) - if err != nil { - return "", fmt.Errorf("creating ec2 client: %w", err) - } - var uefiData *string - if enableSecureBoot { - awsUEFIData, err := uefiVarStore.ToAWS() - if err != nil { - return "", fmt.Errorf("creating uefi data: %w", err) - } - uefiData = toPtr(awsUEFIData) - } - - createReq, err := ec2C.RegisterImage(ctx, &ec2.RegisterImageInput{ - Name: &imageName, - Architecture: ec2types.ArchitectureValuesX8664, - BlockDeviceMappings: []ec2types.BlockDeviceMapping{ - { - DeviceName: toPtr("/dev/xvda"), - Ebs: &ec2types.EbsBlockDevice{ - DeleteOnTermination: toPtr(true), - SnapshotId: &snapshotID, - }, - }, - }, - BootMode: ec2types.BootModeValuesUefi, - Description: toPtr("Constellation " + version.ShortPath()), - EnaSupport: toPtr(true), - RootDeviceName: toPtr("/dev/xvda"), - TpmSupport: ec2types.TpmSupportValuesV20, - UefiData: uefiData, - VirtualizationType: toPtr("hvm"), - }) - if err != nil { - return "", fmt.Errorf("creating image: %w", err) - } - if createReq.ImageId == nil { - return "", fmt.Errorf("creating image: no image ID returned") - } - return *createReq.ImageId, nil -} - -func (u *Uploader) replicateImage(ctx context.Context, imageName, amiID string, region string) (string, error) { - u.log.Debugf("Replicating image %s to %s", imageName, region) - ec2C, err := u.ec2(ctx, region) - if err != nil { - return "", fmt.Errorf("creating ec2 client: %w", err) - } - replicateReq, err := ec2C.CopyImage(ctx, &ec2.CopyImageInput{ - Name: &imageName, - SourceImageId: &amiID, - SourceRegion: &u.region, - }) - if err != nil { - return "", fmt.Errorf("replicating image: %w", err) - } - if replicateReq.ImageId == nil { - return "", fmt.Errorf("replicating image: no image ID returned") - } - return *replicateReq.ImageId, nil -} - -func (u *Uploader) findImage(ctx context.Context, imageName, region string) (string, error) { - ec2C, err := u.ec2(ctx, region) - if err != nil { - return "", fmt.Errorf("creating ec2 client: %w", err) - } - snapshots, err := ec2C.DescribeImages(ctx, &ec2.DescribeImagesInput{ - Filters: []ec2types.Filter{ - { - Name: toPtr("name"), - Values: []string{imageName}, - }, - }, - }) - if err != nil { - return "", fmt.Errorf("describing images: %w", err) - } - if len(snapshots.Images) == 0 { - return "", errAMIDoesNotExist - } - if len(snapshots.Images) != 1 { - return "", fmt.Errorf("expected 1 image, got %d", len(snapshots.Images)) - } - if snapshots.Images[0].ImageId == nil { - return "", fmt.Errorf("image ID is nil") - } - return *snapshots.Images[0].ImageId, nil -} - -func (u *Uploader) waitForImage(ctx context.Context, amiID, region string) error { - u.log.Debugf("Waiting for image %s in %s to be created", amiID, region) - ec2C, err := u.ec2(ctx, region) - if err != nil { - return fmt.Errorf("creating ec2 client: %w", err) - } - waiter := ec2.NewImageAvailableWaiter(ec2C) - err = waiter.Wait(ctx, &ec2.DescribeImagesInput{ - ImageIds: []string{amiID}, - }, maxWait) - if err != nil { - return fmt.Errorf("waiting for image: %w", err) - } - return nil -} - -func (u *Uploader) tagImageAndSnapshot(ctx context.Context, imageName, amiID, region string) error { - u.log.Debugf("Tagging backing snapshot of image %s in %s", amiID, region) - ec2C, err := u.ec2(ctx, region) - if err != nil { - return fmt.Errorf("creating ec2 client: %w", err) - } - snapshotID, err := getBackingSnapshotID(ctx, ec2C, amiID) - if err != nil { - return fmt.Errorf("getting backing snapshot ID: %w", err) - } - _, err = ec2C.CreateTags(ctx, &ec2.CreateTagsInput{ - Resources: []string{amiID, snapshotID}, - Tags: []ec2types.Tag{ - { - Key: toPtr("Name"), - Value: toPtr(imageName), - }, - }, - }) - if err != nil { - return fmt.Errorf("tagging ami and snapshot: %w", err) - } - return nil -} - -func (u *Uploader) publishImage(ctx context.Context, imageName, region string) error { - u.log.Debugf("Publishing image %s in %s", imageName, region) - ec2C, err := u.ec2(ctx, region) - if err != nil { - return fmt.Errorf("creating ec2 client: %w", err) - } - _, err = ec2C.ModifyImageAttribute(ctx, &ec2.ModifyImageAttributeInput{ - ImageId: &imageName, - LaunchPermission: &ec2types.LaunchPermissionModifications{ - Add: []ec2types.LaunchPermission{ - { - Group: ec2types.PermissionGroupAll, - }, - }, - }, - }) - if err != nil { - return fmt.Errorf("publishing image: %w", err) - } - return nil -} - -func (u *Uploader) ensureImageDeleted(ctx context.Context, imageName, region string) error { - ec2C, err := u.ec2(ctx, region) - if err != nil { - return fmt.Errorf("creating ec2 client: %w", err) - } - amiID, err := u.findImage(ctx, imageName, region) - if err == errAMIDoesNotExist { - u.log.Debugf("Image %s in %s doesn't exist. Nothing to clean up.", imageName, region) - return nil - } - snapshotID, err := getBackingSnapshotID(ctx, ec2C, amiID) - if err == errAMIDoesNotExist { - u.log.Debugf("Image %s doesn't exist. Nothing to clean up.", amiID) - return nil - } - u.log.Debugf("Deleting image %s in %s with backing snapshot", amiID, region) - _, err = ec2C.DeregisterImage(ctx, &ec2.DeregisterImageInput{ - ImageId: &amiID, - }) - if err != nil { - return fmt.Errorf("deleting image: %w", err) - } - _, err = ec2C.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{ - SnapshotId: &snapshotID, - }) - if err != nil { - return fmt.Errorf("deleting snapshot: %w", err) - } - return nil -} - -func imageName(version versionsapi.Version, attestationVariant string, timestamp time.Time) string { - if version.Stream() == "stable" { - return fmt.Sprintf("constellation-%s-%s", version.Version(), attestationVariant) - } - return fmt.Sprintf("constellation-%s-%s-%s-%s", version.Stream(), version.Version(), attestationVariant, timestamp.Format(timestampFormat)) -} - -func waitForSnapshotImport(ctx context.Context, ec2C ec2API, importTaskID string) (string, error) { - for { - taskResp, err := ec2C.DescribeImportSnapshotTasks(ctx, &ec2.DescribeImportSnapshotTasksInput{ - ImportTaskIds: []string{importTaskID}, - }) - if err != nil { - return "", fmt.Errorf("describing import snapshot task: %w", err) - } - if len(taskResp.ImportSnapshotTasks) == 0 { - return "", fmt.Errorf("describing import snapshot task: no tasks returned") - } - if taskResp.ImportSnapshotTasks[0].SnapshotTaskDetail == nil { - return "", fmt.Errorf("describing import snapshot task: no snapshot task detail returned") - } - if taskResp.ImportSnapshotTasks[0].SnapshotTaskDetail.Status == nil { - return "", fmt.Errorf("describing import snapshot task: no status returned") - } - switch *taskResp.ImportSnapshotTasks[0].SnapshotTaskDetail.Status { - case string(ec2types.SnapshotStateCompleted): - return *taskResp.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId, nil - case string(ec2types.SnapshotStateError): - return "", fmt.Errorf("importing snapshot: task failed") - } - time.Sleep(waitInterval) - } -} - -func getBackingSnapshotID(ctx context.Context, ec2C ec2API, amiID string) (string, error) { - describeResp, err := ec2C.DescribeImages(ctx, &ec2.DescribeImagesInput{ - ImageIds: []string{amiID}, - }) - if err != nil || len(describeResp.Images) == 0 { - return "", errAMIDoesNotExist - } - if len(describeResp.Images) != 1 { - return "", fmt.Errorf("describing image: expected 1 image, got %d", len(describeResp.Images)) - } - image := describeResp.Images[0] - if len(image.BlockDeviceMappings) != 1 { - return "", fmt.Errorf("found %d block device mappings for image %s, expected 1", len(image.BlockDeviceMappings), amiID) - } - if image.BlockDeviceMappings[0].Ebs == nil { - return "", fmt.Errorf("image %s does not have an EBS block device mapping", amiID) - } - ebs := image.BlockDeviceMappings[0].Ebs - if ebs.SnapshotId == nil { - return "", fmt.Errorf("image %s does not have an EBS snapshot", amiID) - } - return *ebs.SnapshotId, nil -} - -type ec2API interface { - DescribeImages(ctx context.Context, params *ec2.DescribeImagesInput, - optFns ...func(*ec2.Options), - ) (*ec2.DescribeImagesOutput, error) - ModifyImageAttribute(ctx context.Context, params *ec2.ModifyImageAttributeInput, - optFns ...func(*ec2.Options), - ) (*ec2.ModifyImageAttributeOutput, error) - RegisterImage(ctx context.Context, params *ec2.RegisterImageInput, - optFns ...func(*ec2.Options), - ) (*ec2.RegisterImageOutput, error) - CopyImage(ctx context.Context, params *ec2.CopyImageInput, optFns ...func(*ec2.Options), - ) (*ec2.CopyImageOutput, error) - DeregisterImage(ctx context.Context, params *ec2.DeregisterImageInput, - optFns ...func(*ec2.Options), - ) (*ec2.DeregisterImageOutput, error) - ImportSnapshot(ctx context.Context, params *ec2.ImportSnapshotInput, - optFns ...func(*ec2.Options), - ) (*ec2.ImportSnapshotOutput, error) - DescribeImportSnapshotTasks(ctx context.Context, params *ec2.DescribeImportSnapshotTasksInput, - optFns ...func(*ec2.Options), - ) (*ec2.DescribeImportSnapshotTasksOutput, error) - DescribeSnapshots(ctx context.Context, params *ec2.DescribeSnapshotsInput, - optFns ...func(*ec2.Options), - ) (*ec2.DescribeSnapshotsOutput, error) - DeleteSnapshot(ctx context.Context, params *ec2.DeleteSnapshotInput, optFns ...func(*ec2.Options), - ) (*ec2.DeleteSnapshotOutput, error) - CreateTags(ctx context.Context, params *ec2.CreateTagsInput, optFns ...func(*ec2.Options), - ) (*ec2.CreateTagsOutput, error) -} - -type s3API interface { - HeadBucket(ctx context.Context, params *s3.HeadBucketInput, optFns ...func(*s3.Options), - ) (*s3.HeadBucketOutput, error) - CreateBucket(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options), - ) (*s3.CreateBucketOutput, error) - HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options), - ) (*s3.HeadObjectOutput, error) - DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options), - ) (*s3.DeleteObjectOutput, error) -} - -type s3UploaderAPI interface { - Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*s3manager.Uploader), - ) (*s3manager.UploadOutput, error) -} - -func toPtr[T any](v T) *T { - return &v -} - -const ( - waitInterval = 15 * time.Second - maxWait = 30 * time.Minute - timestampFormat = "20060102150405" -) - -var ( - errAMIDoesNotExist = errors.New("ami does not exist") - replicationRegions = []string{"eu-west-1", "eu-west-3", "us-east-2", "ap-south-1"} -) diff --git a/internal/osimage/azure/BUILD.bazel b/internal/osimage/azure/BUILD.bazel deleted file mode 100644 index 48b1b6e77..000000000 --- a/internal/osimage/azure/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "azure", - srcs = [ - "azureupload.go", - "disktype_string.go", - ], - importpath = "github.com/edgelesssys/constellation/v2/internal/osimage/azure", - visibility = ["//:__subpackages__"], - deps = [ - "//internal/api/versionsapi", - "//internal/logger", - "//internal/osimage", - "@com_github_azure_azure_sdk_for_go_sdk_azcore//runtime", - "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity", - "@com_github_azure_azure_sdk_for_go_sdk_resourcemanager_compute_armcompute_v5//:armcompute", - "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//blob", - "@com_github_azure_azure_sdk_for_go_sdk_storage_azblob//pageblob", - ], -) diff --git a/internal/osimage/azure/azureupload.go b/internal/osimage/azure/azureupload.go deleted file mode 100644 index 32490c3ff..000000000 --- a/internal/osimage/azure/azureupload.go +++ /dev/null @@ -1,710 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -// package azure implements uploading os images to azure. -package azure - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "strings" - "time" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/pageblob" - "github.com/edgelesssys/constellation/v2/internal/api/versionsapi" - "github.com/edgelesssys/constellation/v2/internal/logger" - "github.com/edgelesssys/constellation/v2/internal/osimage" -) - -// Uploader can upload and remove os images on Azure. -type Uploader struct { - subscription string - location string - resourceGroup string - pollingFrequency time.Duration - disks azureDiskAPI - managedImages azureManagedImageAPI - blob sasBlobUploader - galleries azureGalleriesAPI - image azureGalleriesImageAPI - imageVersions azureGalleriesImageVersionAPI - communityVersions azureCommunityGalleryImageVersionAPI - - log *logger.Logger -} - -// New creates a new Uploader. -func New(subscription, location, resourceGroup string, log *logger.Logger) (*Uploader, error) { - cred, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - return nil, err - } - diskClient, err := armcomputev5.NewDisksClient(subscription, cred, nil) - if err != nil { - return nil, err - } - managedImagesClient, err := armcomputev5.NewImagesClient(subscription, cred, nil) - if err != nil { - return nil, err - } - galleriesClient, err := armcomputev5.NewGalleriesClient(subscription, cred, nil) - if err != nil { - return nil, err - } - galleriesImageClient, err := armcomputev5.NewGalleryImagesClient(subscription, cred, nil) - if err != nil { - return nil, err - } - galleriesImageVersionClient, err := armcomputev5.NewGalleryImageVersionsClient(subscription, cred, nil) - if err != nil { - return nil, err - } - communityImageVersionClient, err := armcomputev5.NewCommunityGalleryImageVersionsClient(subscription, cred, nil) - if err != nil { - return nil, err - } - - return &Uploader{ - subscription: subscription, - location: location, - resourceGroup: resourceGroup, - pollingFrequency: pollingFrequency, - disks: diskClient, - managedImages: managedImagesClient, - blob: func(sasBlobURL string) (azurePageblobAPI, error) { - return pageblob.NewClientWithNoCredential(sasBlobURL, nil) - }, - galleries: galleriesClient, - image: galleriesImageClient, - imageVersions: galleriesImageVersionClient, - communityVersions: communityImageVersionClient, - log: log, - }, nil -} - -// Upload uploads an OS image to Azure. -func (u *Uploader) Upload(ctx context.Context, req *osimage.UploadRequest) ([]versionsapi.ImageInfoEntry, error) { - formattedTime := req.Timestamp.Format(timestampFormat) - diskName := fmt.Sprintf("constellation-%s-%s-%s", req.Version.Stream(), formattedTime, req.AttestationVariant) - var sigName string - switch req.Version.Stream() { - case "stable": - sigName = sigNameStable - case "debug": - sigName = sigNameDebug - default: - sigName = sigNameDefault - } - definitionName := imageOffer(req.Version) - versionName, err := imageVersion(req.Version, req.Timestamp) - if err != nil { - return nil, fmt.Errorf("determining image version name: %w", err) - } - - // ensure new image can be uploaded by deleting existing resources using the same name - if err := u.ensureImageVersionDeleted(ctx, sigName, definitionName, versionName); err != nil { - return nil, fmt.Errorf("pre-cleaning: ensuring no image version using the same name exists: %w", err) - } - if err := u.ensureManagedImageDeleted(ctx, diskName); err != nil { - return nil, fmt.Errorf("pre-cleaning: ensuring no managed image using the same name exists: %w", err) - } - if err := u.ensureDiskDeleted(ctx, diskName); err != nil { - return nil, fmt.Errorf("pre-cleaning: ensuring no temporary disk using the same name exists: %w", err) - } - - diskID, err := u.createDisk(ctx, diskName, DiskTypeNormal, req.Image, nil, req.Size) - if err != nil { - return nil, fmt.Errorf("creating disk: %w", err) - } - defer func() { - // cleanup temp disk - err := u.ensureDiskDeleted(ctx, diskName) - if err != nil { - u.log.Errorf("post-cleaning: deleting disk image: %v", err) - } - }() - managedImageID, err := u.createManagedImage(ctx, diskName, diskID) - if err != nil { - return nil, fmt.Errorf("creating managed image: %w", err) - } - if err := u.ensureSIG(ctx, sigName); err != nil { - return nil, fmt.Errorf("ensuring sig exists: %w", err) - } - if err := u.ensureImageDefinition(ctx, sigName, definitionName, req.Version, req.AttestationVariant); err != nil { - return nil, fmt.Errorf("ensuring image definition exists: %w", err) - } - - unsharedImageVersionID, err := u.createImageVersion(ctx, sigName, definitionName, versionName, managedImageID) - if err != nil { - return nil, fmt.Errorf("creating image version: %w", err) - } - - imageReference, err := u.getImageReference(ctx, sigName, definitionName, versionName, unsharedImageVersionID) - if err != nil { - return nil, fmt.Errorf("getting image reference: %w", err) - } - - return []versionsapi.ImageInfoEntry{ - { - CSP: "azure", - AttestationVariant: req.AttestationVariant, - Reference: imageReference, - }, - }, nil -} - -// createDisk creates and initializes (uploads contents of) an azure disk. -func (u *Uploader) createDisk(ctx context.Context, diskName string, diskType DiskType, img io.ReadSeeker, vmgs io.ReadSeeker, size int64) (string, error) { - u.log.Debugf("Creating disk %s in %s", diskName, u.resourceGroup) - if diskType == DiskTypeWithVMGS && vmgs == nil { - return "", errors.New("cannot create disk with vmgs: vmgs reader is nil") - } - - var createOption armcomputev5.DiskCreateOption - var requestVMGSSAS bool - switch diskType { - case DiskTypeNormal: - createOption = armcomputev5.DiskCreateOptionUpload - case DiskTypeWithVMGS: - createOption = armcomputev5.DiskCreateOptionUploadPreparedSecure - requestVMGSSAS = true - } - disk := armcomputev5.Disk{ - Location: &u.location, - Properties: &armcomputev5.DiskProperties{ - CreationData: &armcomputev5.CreationData{ - CreateOption: &createOption, - UploadSizeBytes: toPtr(size), - }, - HyperVGeneration: toPtr(armcomputev5.HyperVGenerationV2), - OSType: toPtr(armcomputev5.OperatingSystemTypesLinux), - }, - } - createPoller, err := u.disks.BeginCreateOrUpdate(ctx, u.resourceGroup, diskName, disk, &armcomputev5.DisksClientBeginCreateOrUpdateOptions{}) - if err != nil { - return "", fmt.Errorf("creating disk: %w", err) - } - createdDisk, err := createPoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}) - if err != nil { - return "", fmt.Errorf("waiting for disk to be created: %w", err) - } - - u.log.Debugf("Granting temporary upload permissions via SAS token") - accessGrant := armcomputev5.GrantAccessData{ - Access: toPtr(armcomputev5.AccessLevelWrite), - DurationInSeconds: toPtr(int32(uploadAccessDuration)), - GetSecureVMGuestStateSAS: &requestVMGSSAS, - } - accessPoller, err := u.disks.BeginGrantAccess(ctx, u.resourceGroup, diskName, accessGrant, &armcomputev5.DisksClientBeginGrantAccessOptions{}) - if err != nil { - return "", fmt.Errorf("generating disk sas token: %w", err) - } - accesPollerResp, err := accessPoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}) - if err != nil { - return "", fmt.Errorf("waiting for sas token: %w", err) - } - - if requestVMGSSAS { - u.log.Debugf("Uploading vmgs") - vmgsSize, err := vmgs.Seek(0, io.SeekEnd) - if err != nil { - return "", err - } - if _, err := vmgs.Seek(0, io.SeekStart); err != nil { - return "", err - } - if accesPollerResp.SecurityDataAccessSAS == nil { - return "", errors.New("uploading vmgs: grant access returned no vmgs sas") - } - if err := uploadBlob(ctx, *accesPollerResp.SecurityDataAccessSAS, vmgs, vmgsSize, u.blob); err != nil { - return "", fmt.Errorf("uploading vmgs: %w", err) - } - } - u.log.Debugf("Uploading os image") - if accesPollerResp.AccessSAS == nil { - return "", errors.New("uploading disk: grant access returned no disk sas") - } - if err := uploadBlob(ctx, *accesPollerResp.AccessSAS, img, size, u.blob); err != nil { - return "", fmt.Errorf("uploading image: %w", err) - } - revokePoller, err := u.disks.BeginRevokeAccess(ctx, u.resourceGroup, diskName, &armcomputev5.DisksClientBeginRevokeAccessOptions{}) - if err != nil { - return "", fmt.Errorf("revoking disk sas token: %w", err) - } - if _, err := revokePoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}); err != nil { - return "", fmt.Errorf("waiting for sas token revocation: %w", err) - } - if createdDisk.ID == nil { - return "", errors.New("created disk has no id") - } - return *createdDisk.ID, nil -} - -func (u *Uploader) ensureDiskDeleted(ctx context.Context, diskName string) error { - _, err := u.disks.Get(ctx, u.resourceGroup, diskName, &armcomputev5.DisksClientGetOptions{}) - if err != nil { - u.log.Debugf("Disk %s in %s doesn't exist. Nothing to clean up.", diskName, u.resourceGroup) - return nil - } - u.log.Debugf("Deleting disk %s in %s", diskName, u.resourceGroup) - deletePoller, err := u.disks.BeginDelete(ctx, u.resourceGroup, diskName, &armcomputev5.DisksClientBeginDeleteOptions{}) - if err != nil { - return fmt.Errorf("deleting disk: %w", err) - } - if _, err = deletePoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}); err != nil { - return fmt.Errorf("waiting for disk to be deleted: %w", err) - } - return nil -} - -func (u *Uploader) createManagedImage(ctx context.Context, imageName string, diskID string) (string, error) { - u.log.Debugf("Creating managed image %s in %s", imageName, u.resourceGroup) - image := armcomputev5.Image{ - Location: &u.location, - Properties: &armcomputev5.ImageProperties{ - HyperVGeneration: toPtr(armcomputev5.HyperVGenerationTypesV2), - StorageProfile: &armcomputev5.ImageStorageProfile{ - OSDisk: &armcomputev5.ImageOSDisk{ - OSState: toPtr(armcomputev5.OperatingSystemStateTypesGeneralized), - OSType: toPtr(armcomputev5.OperatingSystemTypesLinux), - ManagedDisk: &armcomputev5.SubResource{ - ID: &diskID, - }, - }, - }, - }, - } - createPoller, err := u.managedImages.BeginCreateOrUpdate( - ctx, u.resourceGroup, imageName, image, - &armcomputev5.ImagesClientBeginCreateOrUpdateOptions{}, - ) - if err != nil { - return "", fmt.Errorf("creating managed image: %w", err) - } - createdImage, err := createPoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}) - if err != nil { - return "", fmt.Errorf("waiting for image to be created: %w", err) - } - if createdImage.ID == nil { - return "", errors.New("created image has no id") - } - return *createdImage.ID, nil -} - -func (u *Uploader) ensureManagedImageDeleted(ctx context.Context, imageName string) error { - _, err := u.managedImages.Get(ctx, u.resourceGroup, imageName, &armcomputev5.ImagesClientGetOptions{}) - if err != nil { - u.log.Debugf("Managed image %s in %s doesn't exist. Nothing to clean up.", imageName, u.resourceGroup) - return nil - } - u.log.Debugf("Deleting managed image %s in %s", imageName, u.resourceGroup) - deletePoller, err := u.managedImages.BeginDelete(ctx, u.resourceGroup, imageName, &armcomputev5.ImagesClientBeginDeleteOptions{}) - if err != nil { - return fmt.Errorf("deleting image: %w", err) - } - if _, err = deletePoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}); err != nil { - return fmt.Errorf("waiting for image to be deleted: %w", err) - } - return nil -} - -// ensureSIG creates a SIG if it does not exist yet. -func (u *Uploader) ensureSIG(ctx context.Context, sigName string) error { - _, err := u.galleries.Get(ctx, u.resourceGroup, sigName, &armcomputev5.GalleriesClientGetOptions{}) - if err == nil { - u.log.Debugf("Image gallery %s in %s exists", sigName, u.resourceGroup) - return nil - } - u.log.Debugf("Creating image gallery %s in %s", sigName, u.resourceGroup) - gallery := armcomputev5.Gallery{ - Location: &u.location, - } - createPoller, err := u.galleries.BeginCreateOrUpdate(ctx, u.resourceGroup, sigName, gallery, - &armcomputev5.GalleriesClientBeginCreateOrUpdateOptions{}, - ) - if err != nil { - return fmt.Errorf("creating image gallery: %w", err) - } - if _, err = createPoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}); err != nil { - return fmt.Errorf("waiting for image gallery to be created: %w", err) - } - return nil -} - -// ensureImageDefinition creates an image definition (component of a SIG) if it does not exist yet. -func (u *Uploader) ensureImageDefinition(ctx context.Context, sigName, definitionName string, version versionsapi.Version, attestationVariant string) error { - _, err := u.image.Get(ctx, u.resourceGroup, sigName, definitionName, &armcomputev5.GalleryImagesClientGetOptions{}) - if err == nil { - u.log.Debugf("Image definition %s/%s in %s exists", sigName, definitionName, u.resourceGroup) - return nil - } - u.log.Debugf("Creating image definition %s/%s in %s", sigName, definitionName, u.resourceGroup) - var securityType string - // TODO(malt3): This needs to allow the *Supported or the normal variant - // based on wether a VMGS was provided or not. - // VMGS provided: ConfidentialVM - // No VMGS provided: ConfidentialVMSupported - switch strings.ToLower(attestationVariant) { - case "azure-sev-snp": - securityType = string("ConfidentialVMSupported") - case "azure-trustedlaunch": - securityType = string(armcomputev5.SecurityTypesTrustedLaunch) - } - offer := imageOffer(version) - - galleryImage := armcomputev5.GalleryImage{ - Location: &u.location, - Properties: &armcomputev5.GalleryImageProperties{ - Identifier: &armcomputev5.GalleryImageIdentifier{ - Offer: &offer, - Publisher: toPtr(imageDefinitionPublisher), - SKU: toPtr(imageDefinitionSKU), - }, - OSState: toPtr(armcomputev5.OperatingSystemStateTypesGeneralized), - OSType: toPtr(armcomputev5.OperatingSystemTypesLinux), - Architecture: toPtr(armcomputev5.ArchitectureX64), - Features: []*armcomputev5.GalleryImageFeature{ - { - Name: toPtr("SecurityType"), - Value: &securityType, - }, - }, - HyperVGeneration: toPtr(armcomputev5.HyperVGenerationV2), - }, - } - createPoller, err := u.image.BeginCreateOrUpdate(ctx, u.resourceGroup, sigName, definitionName, galleryImage, - &armcomputev5.GalleryImagesClientBeginCreateOrUpdateOptions{}, - ) - if err != nil { - return fmt.Errorf("creating image definition: %w", err) - } - if _, err = createPoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}); err != nil { - return fmt.Errorf("waiting for image definition to be created: %w", err) - } - return nil -} - -func (u *Uploader) createImageVersion(ctx context.Context, sigName, definitionName, versionName, imageID string) (string, error) { - u.log.Debugf("Creating image version %s/%s/%s in %s", sigName, definitionName, versionName, u.resourceGroup) - imageVersion := armcomputev5.GalleryImageVersion{ - Location: &u.location, - Properties: &armcomputev5.GalleryImageVersionProperties{ - StorageProfile: &armcomputev5.GalleryImageVersionStorageProfile{ - OSDiskImage: &armcomputev5.GalleryOSDiskImage{ - HostCaching: toPtr(armcomputev5.HostCachingReadOnly), - }, - Source: &armcomputev5.GalleryArtifactVersionFullSource{ - ID: &imageID, - }, - }, - PublishingProfile: &armcomputev5.GalleryImageVersionPublishingProfile{ - ReplicaCount: toPtr[int32](1), - ReplicationMode: toPtr(armcomputev5.ReplicationModeFull), - TargetRegions: targetRegions, - }, - }, - } - createPoller, err := u.imageVersions.BeginCreateOrUpdate(ctx, u.resourceGroup, sigName, definitionName, versionName, imageVersion, - &armcomputev5.GalleryImageVersionsClientBeginCreateOrUpdateOptions{}, - ) - if err != nil { - return "", fmt.Errorf("creating image version: %w", err) - } - createdImage, err := createPoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}) - if err != nil { - return "", fmt.Errorf("waiting for image version to be created: %w", err) - } - if createdImage.ID == nil { - return "", errors.New("created image has no id") - } - return *createdImage.ID, nil -} - -func (u *Uploader) ensureImageVersionDeleted(ctx context.Context, sigName, definitionName, versionName string) error { - _, err := u.imageVersions.Get(ctx, u.resourceGroup, sigName, definitionName, versionName, &armcomputev5.GalleryImageVersionsClientGetOptions{}) - if err != nil { - u.log.Debugf("Image version %s in %s/%s/%s doesn't exist. Nothing to clean up.", versionName, u.resourceGroup, sigName, definitionName) - return nil - } - u.log.Debugf("Deleting image version %s in %s/%s/%s", versionName, u.resourceGroup, sigName, definitionName) - deletePoller, err := u.imageVersions.BeginDelete(ctx, u.resourceGroup, sigName, definitionName, versionName, &armcomputev5.GalleryImageVersionsClientBeginDeleteOptions{}) - if err != nil { - return fmt.Errorf("deleting image version: %w", err) - } - if _, err = deletePoller.PollUntilDone(ctx, &runtime.PollUntilDoneOptions{Frequency: u.pollingFrequency}); err != nil { - return fmt.Errorf("waiting for image version to be deleted: %w", err) - } - return nil -} - -// getImageReference returns the image reference to use for the image version. -// If the shared image gallery is a community gallery, the community identifier is returned. -// Otherwise, the unshared identifier is returned. -func (u *Uploader) getImageReference(ctx context.Context, sigName, definitionName, versionName, unsharedID string) (string, error) { - galleryResp, err := u.galleries.Get(ctx, u.resourceGroup, sigName, &armcomputev5.GalleriesClientGetOptions{}) - if err != nil { - return "", fmt.Errorf("getting image gallery %s: %w", sigName, err) - } - if galleryResp.Properties == nil || - galleryResp.Properties.SharingProfile == nil || - galleryResp.Properties.SharingProfile.CommunityGalleryInfo == nil || - galleryResp.Properties.SharingProfile.CommunityGalleryInfo.CommunityGalleryEnabled == nil || - !*galleryResp.Properties.SharingProfile.CommunityGalleryInfo.CommunityGalleryEnabled { - u.log.Warnf("Image gallery %s in %s is not shared. Using private identifier", sigName, u.resourceGroup) - return unsharedID, nil - } - if galleryResp.Properties == nil || - galleryResp.Properties.SharingProfile == nil || - galleryResp.Properties.SharingProfile.CommunityGalleryInfo == nil || - galleryResp.Properties.SharingProfile.CommunityGalleryInfo.PublicNames == nil || - len(galleryResp.Properties.SharingProfile.CommunityGalleryInfo.PublicNames) < 1 || - galleryResp.Properties.SharingProfile.CommunityGalleryInfo.PublicNames[0] == nil { - return "", fmt.Errorf("image gallery %s in %s is a community gallery but has no public names", sigName, u.resourceGroup) - } - communityGalleryName := *galleryResp.Properties.SharingProfile.CommunityGalleryInfo.PublicNames[0] - u.log.Debugf("Image gallery %s in %s is shared. Using community identifier in %s", sigName, u.resourceGroup, communityGalleryName) - communityVersionResp, err := u.communityVersions.Get(ctx, u.location, communityGalleryName, - definitionName, versionName, - &armcomputev5.CommunityGalleryImageVersionsClientGetOptions{}, - ) - if err != nil { - return "", fmt.Errorf("getting community image version %s/%s/%s: %w", communityGalleryName, definitionName, versionName, err) - } - if communityVersionResp.Identifier == nil || communityVersionResp.Identifier.UniqueID == nil { - return "", fmt.Errorf("community image version %s/%s/%s has no id", communityGalleryName, definitionName, versionName) - } - return *communityVersionResp.Identifier.UniqueID, nil -} - -func uploadBlob(ctx context.Context, sasURL string, disk io.ReadSeeker, size int64, uploader sasBlobUploader) error { - uploadClient, err := uploader(sasURL) - if err != nil { - return fmt.Errorf("uploading blob: %w", err) - } - var offset int64 - var chunksize int - chunk := make([]byte, pageSizeMax) - var readErr error - for offset < size { - chunksize, readErr = io.ReadAtLeast(disk, chunk, 1) - if readErr != nil { - return fmt.Errorf("reading from disk: %w", err) - } - if err := uploadChunk(ctx, uploadClient, bytes.NewReader(chunk[:chunksize]), offset, int64(chunksize)); err != nil { - return fmt.Errorf("uploading chunk: %w", err) - } - offset += int64(chunksize) - } - return nil -} - -func uploadChunk(ctx context.Context, uploader azurePageblobAPI, chunk io.ReadSeeker, offset, chunksize int64) error { - _, err := uploader.UploadPages(ctx, &readSeekNopCloser{chunk}, blob.HTTPRange{ - Offset: offset, - Count: chunksize, - }, nil) - return err -} - -func imageOffer(version versionsapi.Version) string { - switch { - case version.Stream() == "stable": - return "constellation" - case version.Stream() == "debug" && version.Ref() == "-": - return version.Version() - } - return version.Ref() + "-" + version.Stream() -} - -// imageVersion determines the semantic version string used inside a sig image. -// For releases, the actual semantic version of the image (without leading v) is used (major.minor.patch). -// Otherwise, the version is derived from the commit timestamp. -func imageVersion(version versionsapi.Version, timestamp time.Time) (string, error) { - switch { - case version.Stream() == "stable": - fallthrough - case version.Stream() == "debug" && version.Ref() == "-": - return strings.TrimLeft(version.Version(), "v"), nil - } - - formattedTime := timestamp.Format(timestampFormat) - if len(formattedTime) != len(timestampFormat) { - return "", errors.New("invalid timestamp") - } - // ..